Ejemplo n.º 1
0
 def __init__(self, argv=None):
     if argv is None:
         argv = sys.argv[1:]
     cur_path = os.path.abspath(os.path.curdir)
     self.product_path = cur_path
     options = self.parseArgs(argv)
     self.options = options
     if options.debug:
         level = logging.DEBUG
     else:
         level = logging.INFO
     initLogger(options.log_file, True, level)
     logger.debug('### RUN ProductMan from %s args: %s ' %
                  (cur_path, ' '.join(argv)))
Ejemplo n.º 2
0
 def __init__(self, argv=None):
     if argv is None:
         argv = sys.argv[1:]
     cur_path = os.path.abspath(os.path.curdir)
     self.product_path = cur_path
     options = self.parseArgs(argv)
     self.options = options
     if options.debug:
         level = logging.DEBUG
     else:
         level = logging.INFO
     initLogger(options.log_file, True, level)
     logger.debug('### RUN ProductMan from %s args: %s ' %
                  (cur_path, ' '.join(argv)))
Ejemplo n.º 3
0
def main():
    args, parser = args_parse()
    initLogger(logPath)
    start_time = getTimeInSec()
    myPrint(conf.minio_endpoint)
    try:
        prev_time = start_time
        if args.action == 'check_conn' or args.action == 'all':
            myPrint("Action: check minio connection", 1)
            m = MinioWrapper()
            myPrint(m.bucketExists("my-test-bucket"))
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action check_conn: " + prstr, 11)
        if args.action == 'remove_bucket' or args.action == 'all':
            myPrint("Action: remove_bucket test", 1)
            m = MinioWrapper()
            myPrint(m.deleteBucket())
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action remove_bucket: " + prstr, 11)
        if args.action == 'check_hash' or args.action == 'all':
            myPrint("Action: check_hash test", 1)
            packet_names = getJsonFile(hashCheckPacketsPacket)
            for packet in packet_names:
                myPrint("Packet name: " + packet, 2)
                Migration().checkHash(packet)
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action check_hash: " + prstr, 11)
        if args.action == 'check_records' or args.action == 'all':
            myPrint("Action: check_records", 1)
            total_buckets = getJsonFile(bucketListPath)
            total_packets = getJsonFile(packetListPath)
            total_ignored = getJsonFile(ignoredBucketListPath)
            total_migrated = getJsonFile(migratedPackets)
            myPrint("total_buckets: " + str(len(total_buckets)))
            myPrint("total_packets: " + str(len(total_packets)))
            myPrint("total_ignored: " + str(len(total_ignored)))
            myPrint("total_migrated: " + str(len(total_migrated)))
            myPrint(list(set(total_packets) - set(total_migrated)))
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action check_records: " + prstr, 11)
    except:
        prev_time, prstr = timeDiff(start_time)
        myPrint("Total time taken by the script: " + prstr, 11)
        formatted_lines = traceback.format_exc()
        myPrint(formatted_lines, 13)
        sys.exit(1)
    prev_time, prstr = timeDiff(start_time)
    myPrint("Total time taken by the script: " + prstr, 11)
    return sys.exit(0)
Ejemplo n.º 4
0
def main(ctx, **kwargs):
    import pdb
    pdb.set_trace()
    # pdb.set_trace()
    site_name = kwargs['site_name']
    task_type = kwargs['task_type']
    thread_number = kwargs['thread_number']
    server_ip = kwargs['server_ip']
    server_port = kwargs['server_port']
    proxy_type = kwargs['proxy_type']
    log_level = log_dict.get(kwargs['log_level'], logging.INFO)
    register = kwargs['register']
    login = kwargs['login']

    import pdb
    pdb.set_trace()
    if site_name is None or task_type is None:
        click.echo(ctx.get_help())
        sys.exit(-1)

    # 初始化logger
    log_dir = '%s/%s_%s.log' % (settings.logs_dir_home, site_name, task_type)
    logger = utils.initLogger(log_dir, log_level)
    # 开始执行爬虫程序
    mainCrawler = MainCrawler(logger, site_name, task_type, thread_number,
                              server_ip, server_port, proxy_type, register,
                              login)
    import pdb
    pdb.set_trace()
    try:
        mainCrawler.run()
    except:
        traceback.print_exc()
        utils.send_email(traceback.format_exc())
Ejemplo n.º 5
0
def runWriter(config_file_location):
    with open(config_file_location, 'r') as f:
        config = yaml.load(f, Loader=yaml.SafeLoader)

    logger = initLogger('monitor', level=config['log_level'])

    # init database
    db = getPostgresDBCursor(config['target']['postgres_uri'])
    createDatabaseObjectsSafe(db, logger)
    websites_ids = initDictionaries(db, config['websites'], logger)

    threads = []
    thread_id = 0
    for w in config['websites']:
        x = threading.Thread(
            name='thread {}'.format(w['url']),
            target=processItem,
            args=(thread_id,
                  Writer(kafka_consumer=getKafkaConsumer(
                      kafka_connect=config['kafka_connect']),
                         db_cursor=getPostgresDBCursor(
                             config['target']['postgres_uri']),
                         websites_ids=websites_ids.copy()), logger))
        threads.append(x)
        x.start()
        thread_id += 1
Ejemplo n.º 6
0
 def __init__(self, settings_path=None):
     """
     settings -- full path to settings module
     """
     if settings_path is not None:
         Settings(file_path=settings_path)
     elif os.environ.get("BIONIC_SETTINGS_FILE") is None:
         module_path = os.path.dirname(__file__)
         settings_path  = os.path.join(module_path, "settings.yaml")
         os.environ["BIONIC_SETTINGS_FILE"] = settings_path
         Settings()
     else:
         Settings()
     initLogger()
     global logger
     logger = logging.getLogger("bionic")
Ejemplo n.º 7
0
def main():
    args, parser = args_parse()
    initLogger(logPath)
    start_time = getTimeInSec()
    myPrint(conf.minio_endpoint)
    try:
        prev_time = start_time
        if args.action == 'get_buckets' or args.action == 'all':
            myPrint("Action: get_buckets", 1)
            GetBuckets().run()
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action get_buckets: " + prstr, 11)
        if args.action == 'find_packets' or args.action == 'all':
            myPrint("Action: find_packets", 1)
            FindPackets().run()
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action find_packets: " + prstr, 11)
        if args.action == 'migrate' or args.action == 'all':
            myPrint("Action: migrate", 1)
            Migration().run()
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action migrate: " + prstr, 11)
        if args.action == 'get_records' or args.action == 'all':
            myPrint("Action: get_records", 1)
            m = MinioWrapper()
            objs = m.listObjects(conf.new_bucket_name, False)
            new_objs = []
            for ob in objs:
                new_objs.append(ob.replace("/", ""))
            writeJsonFile(migratedPackets, new_objs)
            myPrint("Total objects level 1 " + str(len(new_objs)))
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action get_records: " + prstr, 11)
    except:
        prev_time, prstr = timeDiff(start_time)
        myPrint("Total time taken by the script: " + prstr, 11)
        formatted_lines = traceback.format_exc()
        myPrint(formatted_lines, 13)
        sys.exit(1)
    prev_time, prstr = timeDiff(start_time)
    myPrint("Total time taken by the script: " + prstr, 11)
    return sys.exit(0)
Ejemplo n.º 8
0
def runMonitoring(config_file_location):
    with open(config_file_location, 'r') as f:
        config = yaml.load(f, Loader=yaml.SafeLoader)

    logger = initLogger('monitor', level=config['log_level'])

    threads = []
    for w in config['websites']:
        x = threading.Thread(
            name='thread {}'.format(w['url']),
            target=processItem,
            args=(Checker(
                website=w,
                kafka_producer=getKafkaProducer(
                    kafka_connect=config['kafka_connect']),
                kafka_topic=config['kafka_connect']['topic'],
                default_interval=config['check_every_seconds_default']),
                  logger))
        threads.append(x)
        x.start()
Ejemplo n.º 9
0
    def __init__(self):
        self.users = {}
        write_queues = {}
        task_queues = {}
        count_queue = Queue(maxsize=settings.count_queue_size)

        mc = settings.getMCInstance(isTxMongo=False)
        self.logger = utils.initLogger('%s/run_data_server_main.log' %
                                       settings.logs_dir_home)
        self.db = DbManager.DbManager(self.logger, mc, write_queues,
                                      task_queues, count_queue)

        # 开一个线程定时清理
        ts = []
        t1 = threading.Thread(target=self.sched_cleanup, args=())
        ts.append(t1)
        for t in ts:
            t.setDaemon(True)
            t.start()

        self.logger.info("__init__ finish")
Ejemplo n.º 10
0
from kafka import KafkaProducer
from json import dumps
from utils import initLogger, getConfigPath, readConfigFile
from apiService import APIService
import time

LOGGER = initLogger("KAFKA_PRODUCER_LOG")
CONFIG_FILE_PATH = getConfigPath()

KAFKA_CONFIG_SETTINGS = readConfigFile(CONFIG_FILE_PATH, "kafka")
PRODUCER_SETTINGS = readConfigFile(CONFIG_FILE_PATH, "producer.settings")
KAFKA_TOPIC = PRODUCER_SETTINGS["topic"]
URL = PRODUCER_SETTINGS["url"]
SLEEP_DURATION = int(PRODUCER_SETTINGS["timeout.seconds"])

producer = KafkaProducer(
    value_serializer=lambda v: dumps(v).encode('utf-8'),
    bootstrap_servers=KAFKA_CONFIG_SETTINGS["bootstrap_servers"])


def produceMessage(filteredJson):
    for coinData in filteredJson:
        LOGGER.info(f"Message is : {coinData}")
        nameCoin = coinData["name_coin"]
        try:
            producer.send(topic=KAFKA_TOPIC, value=coinData)
        except Exception as e:
            LOGGER.error(f"Coin : {nameCoin} has exception : {e}")


if __name__ == "__main__":
Ejemplo n.º 11
0
        for sql in expireSql:
            while True:
                utils.logInfo("execute sql=%s" % sql)
                if logdb.update(sql) == 0:
                    break
    utils.logInfo("all done!")

    # 关闭连接
    logdb.disconnect()
    gmsdb.disconnect()


if __name__ == "__main__":
    props = Properties()
    props.load(open('conf/conf.properties'))
    utils.initLogger(props["log.file.path"])

    parser = OptionParser()
    parser.add_option("-b",
                      "--btime",
                      action="store",
                      type="string",
                      dest="baseTime",
                      default=str(datetime.date.today()),
                      help="基准时间")
    parser.add_option("-g",
                      "--srvs",
                      action="store",
                      type="string",
                      dest="srvs",
                      default=props["srvs"],
Ejemplo n.º 12
0
def setup_module(module):
    logger = initLogger('monitor', level='DEBUG')
Ejemplo n.º 13
0
def main(argv):
	utils.initLogger(values.LOG_FILE, argv)
	utils.registerInterrupt(cleanup)    
	
	#Initialize (values.PIN is board.PIN, gpiozero needs pin number, so call board.PIN.id)
	global rxThread, statusThread
	status_q = queue.Queue()
	statusThread = status.StatusThread(status_q, 
								values.STATUS_PIN1.id,
								values.STATUS_PIN2.id,
								values.STATUS_PIN3.id)
	statusThread.start()

	
	#Initialize Navigation
	logger.info("Initializing Navigation")
	nav = autopilot.Navigation(status_q, values.TURN_DEBOUNCE_SECS)

	#Initialize Relays
	logger.info("Initializing Controls")
	controls = ctl.Control(status_q, values.MAX_TURN_TIME_SECS)
	controls.direction = ctl.Direction(
										values.TURN_MASTER_RELAY_PIN.id,
										values.TURN_POWER_RELAY_PIN.id,
										values.TURN_GROUND_RELAY_PIN.id)
	controls.speed = ctl.Speed(status_q,
									values.SPEED_MASTER_RELAY_PIN.id,
									values.SPEED_R1_RELAY_PIN.id,
									values.SPEED_R2_RELAY_PIN.id,
									values.SPEED_R3_RELAY_PIN.id,
									values.SPEED_R4_RELAY_PIN.id)
	controls.stop()

	#Initialize RF Remote
	logger.info("Initializing RF Receiver")
	q = queue.Queue()
	rxThread = rfcontrols.RxThread(status_q, values.RX_PIN.id, rfcontrols.Yosoo4ButtonRemote(), q)


	#Pause for hardware to initialize
	time.sleep(2)

	#Start Threads
	logger.info("Starting Remote Thread")
	rxThread.start()
	
	#Begin Keep Main Thread open for interrupt signal
	stopDownAt = 0
	goDownAt = 0
	while True:
		#Read Navigation Data
		nav.read()

		#Check for remote control button events
		try:
			m = q.get_nowait()
			logger.debug(f"Button Event: {m}")
			button = m["button"]
			if m["position"] == rfcontrols.BUTTON_DOWN:
				if button.id == rfcontrols.GO_BTN:
					goDownAt = time.time()
					stopDownAt = 0
					controls.speed.bump()
				elif button.id == rfcontrols.STOP_BTN:
					goDownAt = 0
					stopDownAt = time.time()
					controls.speed.bump(-1)
				elif button.id == rfcontrols.LEFT_BTN:
					controls.turnLeft()
				elif button.id == rfcontrols.RIGHT_BTN:
					controls.turnRight()
				else:
					logger.warning(f"Unsupported button {button.id}")
			else:
				if button.id == rfcontrols.GO_BTN:
					goDownAt = 0
				elif button.id == rfcontrols.STOP_BTN:
					stopDownAt = 0
				elif button.id == rfcontrols.LEFT_BTN:
					controls.stopTurn()
				elif button.id == rfcontrols.RIGHT_BTN:
					controls.stopTurn()
				else:
					logger.warning(f"Unsupported button {button.id}")
		except queue.Empty:
			pass

		#Check for long presses on remote control button events
		if goDownAt > 0:
			dur = time.time() - goDownAt
			if dur > 2:
				logger.info("Go Long Press")
				nav.setHeadingLock()
			if dur > 5:
				logger.info("Go Extended Long Press")
				controls.stop()
				nav.setAnchorLock()
		elif stopDownAt > 0:
			dur = time.time() - stopDownAt
			if dur > 2:
				logger.info("Stop Long Press")
				nav.setHeadingLock(False)
				nav.setAnchorLock(False)
				controls.stop()
				controls.speed.set(ctl.Speed.MIN)
			if dur > 10:
				logger.info("Stop Extended Long Press")
				controls.resetTurnHeading()
			
		#Check autopilot needs
		nav.applyCoarseCorrection(controls)

		#Check that turning is not stuck
		controls.checkTurn()

		time.sleep(0.25)
Ejemplo n.º 14
0
import requests
from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
from utils import currentUnixTime, initLogger

LOGGER = initLogger("API_SERVICE_LOG")


class APIService:
    def getJson(self, url):
        try:
            response = requests.get(url)
            statusCode = response.status_code
            assert statusCode == 200, f"Response status code is : {statusCode}"
            LOGGER.info("Received JSON response")
            return response.json()
        except (AssertionError, ConnectionError, Timeout,
                TooManyRedirects) as e:
            LOGGER.error(f"Exception occurred : {e}")
            return {}

    def parseCoinData(self, coinData):
        try:
            return {
                "name_coin": coinData["name"],
                "symbol_coin": coinData["symbol"],
                "id": coinData["id"],
                "uuid": coinData["uuid"],
                "number_of_markets": coinData["numberOfMarkets"],
                "volume": coinData["volume"],
                "market_cap": coinData["marketCap"],
                "total_supply": coinData["totalSupply"],
Ejemplo n.º 15
0
import unittest
from paths import envPath, logPath
from dotenv import load_dotenv
load_dotenv()

# OR, the same with increased verbosity
load_dotenv(verbose=True)

load_dotenv(dotenv_path=envPath)

from minioWrapper import MinioWrapper
from utils import ridToCenterTimestamp, initLogger, myPrint
initLogger(logPath)


class MyTestCase(unittest.TestCase):
    def test_listBuckets(self):
        print("OKay")
        m = MinioWrapper()
        myPrint(m.listBuckets())

    def test_bucketExists(self):
        m = MinioWrapper()
        myPrint(m.bucketExists("10001100010002420210223073024"))

    def test_getObject(self):
        m = MinioWrapper()
        objects = [
            'RESIDENT/RES_UPDATE/10001100010002420210223073024_evidence',
            'RESIDENT/RES_UPDATE/10001100010002420210223073024_id',
            'RESIDENT/RES_UPDATE/10001100010002420210223073024_optional'
Ejemplo n.º 16
0
    # [(name, list(group)) for name, group in itertools.groupby(qs, lambda p:p['date'])]


class test1:
    t1 = 2
    t2 = "a"

    def aa(self):
        def f1():
            print self.t1

        f1()


if __name__ == "__main__":
    utils.initLogger(None)
    parser = OptionParser()
    parser.add_option("-o",
                      "--opt",
                      action="store",
                      type="string",
                      dest="opt",
                      default="hd",
                      help="渠道标示")
    parser.add_option("-b",
                      "--btime",
                      action="store",
                      type="string",
                      dest="baseTime",
                      default=str(datetime.date.today()),
                      help="基准时间")
Ejemplo n.º 17
0
    for k, v in managerEmails.items():
        emailMatch[k] = {}
        emailMatch[k]['managerEmails'] = []
        emailMatch[k]['agents'] = []

    for obj in objects:
        if getattr(obj, 'managerNotify', None) is not None:
            if obj.managerNotify in emailMatch.keys():
                emailMatch[obj.managerNotify]['managerEmails'].append([a for a in obj.managerEmails \
                                    if a not in emailMatch[obj.managerNotify]['managerEmails']])
                emailMatch[obj.managerNotify]['agents'].append(obj.name)
                continue
    return emailMatch


logging = initLogger()


def main():

    # At several points we are going to turn these vars to false if there is an error or
    # unexpected results. Doing so will prevent any emails from going out.
    proceedToEmailMangers, proceedToEmailAgents = True, True
    proceedManagerReasons, proceedAgentReasons = [], []

    # These events have already results in a warning email to the agent. The
    # event IDs were stored in a txt file from prior runs of the script.
    calendarEventsAlreadyNotified, retrieveCalFuncStatus = retrieveCalendarEventsAlreadyNotified(
    )
    logging.info('Retrieved {} events from list of previously reminded PTO events.'.format(\
                                                           len(calendarEventsAlreadyNotified)))
Ejemplo n.º 18
0
def main():
    args, parser = args_parse()
    initLogger(logPath)
    start_time = getTimeInSec()
    db = DatabaseSession(conf.db_host, conf.db_port, conf.db_user,
                         conf.db_pass)
    try:
        prev_time = start_time
        if args.action == 'get_vids' or args.action == 'all':
            myPrint("Action: get_vids", 1)
            vids = []
            vid_dicts = db.getVids()
            for vid_dict in vid_dicts:
                vids.append(vid_dict['vid'])
            writeJsonFile(vidListPath, vids)
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action get_vids: " + prstr, 11)
        if args.action == 'fetch_info' or args.action == 'all':
            output = []
            myPrint("Action: fetch_info", 1)
            ms = MosipSession(conf.server, conf.regproc_client_id,
                              conf.regproc_secret_key, conf.regproc_app_id)
            vids = getJsonFile(vidListPath)
            for vid in vids:
                myPrint("Operating on VID " + vid, 3)
                res = ms.getUin(vid)
                uin = res['identity']['UIN']
                if conf.debug:
                    myPrint("UIN: " + uin)

                modulo = int(uin) % conf.idrepo_modulo
                myPrint("Modulo: " + str(modulo))
                salt_row = db.getHash(modulo)
                if salt_row is not None:
                    salt = salt_row['salt']
                    if conf.debug:
                        myPrint("Salt: " + salt)
                    uin_hash = hashlib.sha256(bytes(uin + salt,
                                                    'utf-8')).hexdigest()
                    mod_uin_hash = str(modulo) + "_" + uin_hash.upper()
                    rid_row = db.getRid(mod_uin_hash)
                    if rid_row is not None:
                        rid = rid_row['rid']
                        myPrint("RID found")
                        if conf.debug:
                            myPrint("RID: " + rid)
                        center_id, timestamp = ridToCenterTimestamp(rid)
                        output.append({
                            'vid': vid,
                            'uin': uin,
                            'mod_uin_hash': mod_uin_hash,
                            'salt': salt,
                            'rid': rid,
                            'center_id': center_id,
                            'timestamp': timestamp
                        })
                    else:
                        raise RuntimeError(
                            "RID not found for for mod_uin_hash: " +
                            mod_uin_hash)
                else:
                    raise RuntimeError("salt not found for for modulo: " +
                                       str(modulo))
            writeJsonFile(credentialPreparedDataPath, output)
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action fetch_info: " + prstr, 11)

        if args.action == 'reprint' or args.action == 'all':
            myPrint("Action: reprint", 1)
            output = []
            vids = getJsonFile(credentialPreparedDataPath)
            ms = MosipSession(conf.server, conf.ida_client_id,
                              conf.ida_secret_key, conf.ida_app_id)
            for vidInfo in vids:
                myPrint("VID: " + vidInfo['vid'], 3)
                data = {
                    "id": vidInfo['vid'],
                    "credentialType": conf.credential_type,
                    "issuer": conf.partner_id,
                    "recepiant": "",
                    "user": "******",
                    "encrypt": False,
                    "encryptionKey": "",
                    "sharableAttributes": [],
                    "additionalData": {
                        'centerId': vidInfo['center_id'],
                        'creationDate': vidInfo['timestamp'],
                        'registrationId': vidInfo['rid']
                    }
                }
                json_data = json.dumps(data, separators=(',', ':'))
                myPrint(json_data)
                if db.checkRequestInCredentialTransaction(json_data) is None:
                    resp = ms.credentialRequest(data)
                    output.append(resp)
                else:
                    myPrint("Skipping credential request", 11)
            writeJsonFile(vidRequestId, output)
            myPrint('Input VIDs: ' + str(len(vids)), 12)
            myPrint('Output RequestIds: ' + str(len(output)), 12)
            prev_time, prstr = timeDiff(prev_time)
            myPrint("Time taken by Action reprint: " + prstr, 11)
        db.closeAll()
    except:
        db.closeAll()
        prev_time, prstr = timeDiff(start_time)
        myPrint("Total time taken by the script: " + prstr, 11)
        formatted_lines = traceback.format_exc()
        myPrint(formatted_lines, 13)
        sys.exit(1)
    prev_time, prstr = timeDiff(start_time)
    myPrint("Total time taken by the script: " + prstr, 11)
    return sys.exit(0)