def __init__(self, argv=None):
if argv is None:
argv = sys.argv[1:]
cur_path = os.path.abspath(os.path.curdir)
self.product_path = cur_path
options = self.parseArgs(argv)
self.options = options
if options.debug:
level = logging.DEBUG
else:
level = logging.INFO
initLogger(options.log_file, True, level)
logger.debug('### RUN ProductMan from %s args: %s ' %
(cur_path, ' '.join(argv)))
def __init__(self, argv=None):
if argv is None:
argv = sys.argv[1:]
cur_path = os.path.abspath(os.path.curdir)
self.product_path = cur_path
options = self.parseArgs(argv)
self.options = options
if options.debug:
level = logging.DEBUG
else:
level = logging.INFO
initLogger(options.log_file, True, level)
logger.debug('### RUN ProductMan from %s args: %s ' %
(cur_path, ' '.join(argv)))
def main():
args, parser = args_parse()
initLogger(logPath)
start_time = getTimeInSec()
myPrint(conf.minio_endpoint)
try:
prev_time = start_time
if args.action == 'check_conn' or args.action == 'all':
myPrint("Action: check minio connection", 1)
m = MinioWrapper()
myPrint(m.bucketExists("my-test-bucket"))
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action check_conn: " + prstr, 11)
if args.action == 'remove_bucket' or args.action == 'all':
myPrint("Action: remove_bucket test", 1)
m = MinioWrapper()
myPrint(m.deleteBucket())
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action remove_bucket: " + prstr, 11)
if args.action == 'check_hash' or args.action == 'all':
myPrint("Action: check_hash test", 1)
packet_names = getJsonFile(hashCheckPacketsPacket)
for packet in packet_names:
myPrint("Packet name: " + packet, 2)
Migration().checkHash(packet)
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action check_hash: " + prstr, 11)
if args.action == 'check_records' or args.action == 'all':
myPrint("Action: check_records", 1)
total_buckets = getJsonFile(bucketListPath)
total_packets = getJsonFile(packetListPath)
total_ignored = getJsonFile(ignoredBucketListPath)
total_migrated = getJsonFile(migratedPackets)
myPrint("total_buckets: " + str(len(total_buckets)))
myPrint("total_packets: " + str(len(total_packets)))
myPrint("total_ignored: " + str(len(total_ignored)))
myPrint("total_migrated: " + str(len(total_migrated)))
myPrint(list(set(total_packets) - set(total_migrated)))
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action check_records: " + prstr, 11)
except:
prev_time, prstr = timeDiff(start_time)
myPrint("Total time taken by the script: " + prstr, 11)
formatted_lines = traceback.format_exc()
myPrint(formatted_lines, 13)
sys.exit(1)
prev_time, prstr = timeDiff(start_time)
myPrint("Total time taken by the script: " + prstr, 11)
return sys.exit(0)
def main(ctx, **kwargs):
import pdb
pdb.set_trace()
# pdb.set_trace()
site_name = kwargs['site_name']
task_type = kwargs['task_type']
thread_number = kwargs['thread_number']
server_ip = kwargs['server_ip']
server_port = kwargs['server_port']
proxy_type = kwargs['proxy_type']
log_level = log_dict.get(kwargs['log_level'], logging.INFO)
register = kwargs['register']
login = kwargs['login']
import pdb
pdb.set_trace()
if site_name is None or task_type is None:
click.echo(ctx.get_help())
sys.exit(-1)
# 初始化logger
log_dir = '%s/%s_%s.log' % (settings.logs_dir_home, site_name, task_type)
logger = utils.initLogger(log_dir, log_level)
# 开始执行爬虫程序
mainCrawler = MainCrawler(logger, site_name, task_type, thread_number,
server_ip, server_port, proxy_type, register,
login)
import pdb
pdb.set_trace()
try:
mainCrawler.run()
except:
traceback.print_exc()
utils.send_email(traceback.format_exc())
def runWriter(config_file_location):
with open(config_file_location, 'r') as f:
config = yaml.load(f, Loader=yaml.SafeLoader)
logger = initLogger('monitor', level=config['log_level'])
# init database
db = getPostgresDBCursor(config['target']['postgres_uri'])
createDatabaseObjectsSafe(db, logger)
websites_ids = initDictionaries(db, config['websites'], logger)
threads = []
thread_id = 0
for w in config['websites']:
x = threading.Thread(
name='thread {}'.format(w['url']),
target=processItem,
args=(thread_id,
Writer(kafka_consumer=getKafkaConsumer(
kafka_connect=config['kafka_connect']),
db_cursor=getPostgresDBCursor(
config['target']['postgres_uri']),
websites_ids=websites_ids.copy()), logger))
threads.append(x)
x.start()
thread_id += 1
def __init__(self, settings_path=None):
"""
settings -- full path to settings module
"""
if settings_path is not None:
Settings(file_path=settings_path)
elif os.environ.get("BIONIC_SETTINGS_FILE") is None:
module_path = os.path.dirname(__file__)
settings_path = os.path.join(module_path, "settings.yaml")
os.environ["BIONIC_SETTINGS_FILE"] = settings_path
Settings()
else:
Settings()
initLogger()
global logger
logger = logging.getLogger("bionic")
def main():
args, parser = args_parse()
initLogger(logPath)
start_time = getTimeInSec()
myPrint(conf.minio_endpoint)
try:
prev_time = start_time
if args.action == 'get_buckets' or args.action == 'all':
myPrint("Action: get_buckets", 1)
GetBuckets().run()
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action get_buckets: " + prstr, 11)
if args.action == 'find_packets' or args.action == 'all':
myPrint("Action: find_packets", 1)
FindPackets().run()
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action find_packets: " + prstr, 11)
if args.action == 'migrate' or args.action == 'all':
myPrint("Action: migrate", 1)
Migration().run()
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action migrate: " + prstr, 11)
if args.action == 'get_records' or args.action == 'all':
myPrint("Action: get_records", 1)
m = MinioWrapper()
objs = m.listObjects(conf.new_bucket_name, False)
new_objs = []
for ob in objs:
new_objs.append(ob.replace("/", ""))
writeJsonFile(migratedPackets, new_objs)
myPrint("Total objects level 1 " + str(len(new_objs)))
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action get_records: " + prstr, 11)
except:
prev_time, prstr = timeDiff(start_time)
myPrint("Total time taken by the script: " + prstr, 11)
formatted_lines = traceback.format_exc()
myPrint(formatted_lines, 13)
sys.exit(1)
prev_time, prstr = timeDiff(start_time)
myPrint("Total time taken by the script: " + prstr, 11)
return sys.exit(0)
def runMonitoring(config_file_location):
with open(config_file_location, 'r') as f:
config = yaml.load(f, Loader=yaml.SafeLoader)
logger = initLogger('monitor', level=config['log_level'])
threads = []
for w in config['websites']:
x = threading.Thread(
name='thread {}'.format(w['url']),
target=processItem,
args=(Checker(
website=w,
kafka_producer=getKafkaProducer(
kafka_connect=config['kafka_connect']),
kafka_topic=config['kafka_connect']['topic'],
default_interval=config['check_every_seconds_default']),
logger))
threads.append(x)
x.start()
def __init__(self):
self.users = {}
write_queues = {}
task_queues = {}
count_queue = Queue(maxsize=settings.count_queue_size)
mc = settings.getMCInstance(isTxMongo=False)
self.logger = utils.initLogger('%s/run_data_server_main.log' %
settings.logs_dir_home)
self.db = DbManager.DbManager(self.logger, mc, write_queues,
task_queues, count_queue)
# 开一个线程定时清理
ts = []
t1 = threading.Thread(target=self.sched_cleanup, args=())
ts.append(t1)
for t in ts:
t.setDaemon(True)
t.start()
self.logger.info("__init__ finish")
from kafka import KafkaProducer
from json import dumps
from utils import initLogger, getConfigPath, readConfigFile
from apiService import APIService
import time
LOGGER = initLogger("KAFKA_PRODUCER_LOG")
CONFIG_FILE_PATH = getConfigPath()
KAFKA_CONFIG_SETTINGS = readConfigFile(CONFIG_FILE_PATH, "kafka")
PRODUCER_SETTINGS = readConfigFile(CONFIG_FILE_PATH, "producer.settings")
KAFKA_TOPIC = PRODUCER_SETTINGS["topic"]
URL = PRODUCER_SETTINGS["url"]
SLEEP_DURATION = int(PRODUCER_SETTINGS["timeout.seconds"])
producer = KafkaProducer(
value_serializer=lambda v: dumps(v).encode('utf-8'),
bootstrap_servers=KAFKA_CONFIG_SETTINGS["bootstrap_servers"])
def produceMessage(filteredJson):
for coinData in filteredJson:
LOGGER.info(f"Message is : {coinData}")
nameCoin = coinData["name_coin"]
try:
producer.send(topic=KAFKA_TOPIC, value=coinData)
except Exception as e:
LOGGER.error(f"Coin : {nameCoin} has exception : {e}")
if __name__ == "__main__":
for sql in expireSql:
while True:
utils.logInfo("execute sql=%s" % sql)
if logdb.update(sql) == 0:
break
utils.logInfo("all done!")
# 关闭连接
logdb.disconnect()
gmsdb.disconnect()
if __name__ == "__main__":
props = Properties()
props.load(open('conf/conf.properties'))
utils.initLogger(props["log.file.path"])
parser = OptionParser()
parser.add_option("-b",
"--btime",
action="store",
type="string",
dest="baseTime",
default=str(datetime.date.today()),
help="基准时间")
parser.add_option("-g",
"--srvs",
action="store",
type="string",
dest="srvs",
default=props["srvs"],
def setup_module(module):
logger = initLogger('monitor', level='DEBUG')
def main(argv):
utils.initLogger(values.LOG_FILE, argv)
utils.registerInterrupt(cleanup)
#Initialize (values.PIN is board.PIN, gpiozero needs pin number, so call board.PIN.id)
global rxThread, statusThread
status_q = queue.Queue()
statusThread = status.StatusThread(status_q,
values.STATUS_PIN1.id,
values.STATUS_PIN2.id,
values.STATUS_PIN3.id)
statusThread.start()
#Initialize Navigation
logger.info("Initializing Navigation")
nav = autopilot.Navigation(status_q, values.TURN_DEBOUNCE_SECS)
#Initialize Relays
logger.info("Initializing Controls")
controls = ctl.Control(status_q, values.MAX_TURN_TIME_SECS)
controls.direction = ctl.Direction(
values.TURN_MASTER_RELAY_PIN.id,
values.TURN_POWER_RELAY_PIN.id,
values.TURN_GROUND_RELAY_PIN.id)
controls.speed = ctl.Speed(status_q,
values.SPEED_MASTER_RELAY_PIN.id,
values.SPEED_R1_RELAY_PIN.id,
values.SPEED_R2_RELAY_PIN.id,
values.SPEED_R3_RELAY_PIN.id,
values.SPEED_R4_RELAY_PIN.id)
controls.stop()
#Initialize RF Remote
logger.info("Initializing RF Receiver")
q = queue.Queue()
rxThread = rfcontrols.RxThread(status_q, values.RX_PIN.id, rfcontrols.Yosoo4ButtonRemote(), q)
#Pause for hardware to initialize
time.sleep(2)
#Start Threads
logger.info("Starting Remote Thread")
rxThread.start()
#Begin Keep Main Thread open for interrupt signal
stopDownAt = 0
goDownAt = 0
while True:
#Read Navigation Data
nav.read()
#Check for remote control button events
try:
m = q.get_nowait()
logger.debug(f"Button Event: {m}")
button = m["button"]
if m["position"] == rfcontrols.BUTTON_DOWN:
if button.id == rfcontrols.GO_BTN:
goDownAt = time.time()
stopDownAt = 0
controls.speed.bump()
elif button.id == rfcontrols.STOP_BTN:
goDownAt = 0
stopDownAt = time.time()
controls.speed.bump(-1)
elif button.id == rfcontrols.LEFT_BTN:
controls.turnLeft()
elif button.id == rfcontrols.RIGHT_BTN:
controls.turnRight()
else:
logger.warning(f"Unsupported button {button.id}")
else:
if button.id == rfcontrols.GO_BTN:
goDownAt = 0
elif button.id == rfcontrols.STOP_BTN:
stopDownAt = 0
elif button.id == rfcontrols.LEFT_BTN:
controls.stopTurn()
elif button.id == rfcontrols.RIGHT_BTN:
controls.stopTurn()
else:
logger.warning(f"Unsupported button {button.id}")
except queue.Empty:
pass
#Check for long presses on remote control button events
if goDownAt > 0:
dur = time.time() - goDownAt
if dur > 2:
logger.info("Go Long Press")
nav.setHeadingLock()
if dur > 5:
logger.info("Go Extended Long Press")
controls.stop()
nav.setAnchorLock()
elif stopDownAt > 0:
dur = time.time() - stopDownAt
if dur > 2:
logger.info("Stop Long Press")
nav.setHeadingLock(False)
nav.setAnchorLock(False)
controls.stop()
controls.speed.set(ctl.Speed.MIN)
if dur > 10:
logger.info("Stop Extended Long Press")
controls.resetTurnHeading()
#Check autopilot needs
nav.applyCoarseCorrection(controls)
#Check that turning is not stuck
controls.checkTurn()
time.sleep(0.25)
import requests
from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
from utils import currentUnixTime, initLogger
LOGGER = initLogger("API_SERVICE_LOG")
class APIService:
def getJson(self, url):
try:
response = requests.get(url)
statusCode = response.status_code
assert statusCode == 200, f"Response status code is : {statusCode}"
LOGGER.info("Received JSON response")
return response.json()
except (AssertionError, ConnectionError, Timeout,
TooManyRedirects) as e:
LOGGER.error(f"Exception occurred : {e}")
return {}
def parseCoinData(self, coinData):
try:
return {
"name_coin": coinData["name"],
"symbol_coin": coinData["symbol"],
"id": coinData["id"],
"uuid": coinData["uuid"],
"number_of_markets": coinData["numberOfMarkets"],
"volume": coinData["volume"],
"market_cap": coinData["marketCap"],
"total_supply": coinData["totalSupply"],
import unittest
from paths import envPath, logPath
from dotenv import load_dotenv
load_dotenv()
# OR, the same with increased verbosity
load_dotenv(verbose=True)
load_dotenv(dotenv_path=envPath)
from minioWrapper import MinioWrapper
from utils import ridToCenterTimestamp, initLogger, myPrint
initLogger(logPath)
class MyTestCase(unittest.TestCase):
def test_listBuckets(self):
print("OKay")
m = MinioWrapper()
myPrint(m.listBuckets())
def test_bucketExists(self):
m = MinioWrapper()
myPrint(m.bucketExists("10001100010002420210223073024"))
def test_getObject(self):
m = MinioWrapper()
objects = [
'RESIDENT/RES_UPDATE/10001100010002420210223073024_evidence',
'RESIDENT/RES_UPDATE/10001100010002420210223073024_id',
'RESIDENT/RES_UPDATE/10001100010002420210223073024_optional'
# [(name, list(group)) for name, group in itertools.groupby(qs, lambda p:p['date'])]
class test1:
t1 = 2
t2 = "a"
def aa(self):
def f1():
print self.t1
f1()
if __name__ == "__main__":
utils.initLogger(None)
parser = OptionParser()
parser.add_option("-o",
"--opt",
action="store",
type="string",
dest="opt",
default="hd",
help="渠道标示")
parser.add_option("-b",
"--btime",
action="store",
type="string",
dest="baseTime",
default=str(datetime.date.today()),
help="基准时间")
for k, v in managerEmails.items():
emailMatch[k] = {}
emailMatch[k]['managerEmails'] = []
emailMatch[k]['agents'] = []
for obj in objects:
if getattr(obj, 'managerNotify', None) is not None:
if obj.managerNotify in emailMatch.keys():
emailMatch[obj.managerNotify]['managerEmails'].append([a for a in obj.managerEmails \
if a not in emailMatch[obj.managerNotify]['managerEmails']])
emailMatch[obj.managerNotify]['agents'].append(obj.name)
continue
return emailMatch
logging = initLogger()
def main():
# At several points we are going to turn these vars to false if there is an error or
# unexpected results. Doing so will prevent any emails from going out.
proceedToEmailMangers, proceedToEmailAgents = True, True
proceedManagerReasons, proceedAgentReasons = [], []
# These events have already results in a warning email to the agent. The
# event IDs were stored in a txt file from prior runs of the script.
calendarEventsAlreadyNotified, retrieveCalFuncStatus = retrieveCalendarEventsAlreadyNotified(
)
logging.info('Retrieved {} events from list of previously reminded PTO events.'.format(\
len(calendarEventsAlreadyNotified)))
def main():
args, parser = args_parse()
initLogger(logPath)
start_time = getTimeInSec()
db = DatabaseSession(conf.db_host, conf.db_port, conf.db_user,
conf.db_pass)
try:
prev_time = start_time
if args.action == 'get_vids' or args.action == 'all':
myPrint("Action: get_vids", 1)
vids = []
vid_dicts = db.getVids()
for vid_dict in vid_dicts:
vids.append(vid_dict['vid'])
writeJsonFile(vidListPath, vids)
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action get_vids: " + prstr, 11)
if args.action == 'fetch_info' or args.action == 'all':
output = []
myPrint("Action: fetch_info", 1)
ms = MosipSession(conf.server, conf.regproc_client_id,
conf.regproc_secret_key, conf.regproc_app_id)
vids = getJsonFile(vidListPath)
for vid in vids:
myPrint("Operating on VID " + vid, 3)
res = ms.getUin(vid)
uin = res['identity']['UIN']
if conf.debug:
myPrint("UIN: " + uin)
modulo = int(uin) % conf.idrepo_modulo
myPrint("Modulo: " + str(modulo))
salt_row = db.getHash(modulo)
if salt_row is not None:
salt = salt_row['salt']
if conf.debug:
myPrint("Salt: " + salt)
uin_hash = hashlib.sha256(bytes(uin + salt,
'utf-8')).hexdigest()
mod_uin_hash = str(modulo) + "_" + uin_hash.upper()
rid_row = db.getRid(mod_uin_hash)
if rid_row is not None:
rid = rid_row['rid']
myPrint("RID found")
if conf.debug:
myPrint("RID: " + rid)
center_id, timestamp = ridToCenterTimestamp(rid)
output.append({
'vid': vid,
'uin': uin,
'mod_uin_hash': mod_uin_hash,
'salt': salt,
'rid': rid,
'center_id': center_id,
'timestamp': timestamp
})
else:
raise RuntimeError(
"RID not found for for mod_uin_hash: " +
mod_uin_hash)
else:
raise RuntimeError("salt not found for for modulo: " +
str(modulo))
writeJsonFile(credentialPreparedDataPath, output)
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action fetch_info: " + prstr, 11)
if args.action == 'reprint' or args.action == 'all':
myPrint("Action: reprint", 1)
output = []
vids = getJsonFile(credentialPreparedDataPath)
ms = MosipSession(conf.server, conf.ida_client_id,
conf.ida_secret_key, conf.ida_app_id)
for vidInfo in vids:
myPrint("VID: " + vidInfo['vid'], 3)
data = {
"id": vidInfo['vid'],
"credentialType": conf.credential_type,
"issuer": conf.partner_id,
"recepiant": "",
"user": "******",
"encrypt": False,
"encryptionKey": "",
"sharableAttributes": [],
"additionalData": {
'centerId': vidInfo['center_id'],
'creationDate': vidInfo['timestamp'],
'registrationId': vidInfo['rid']
}
}
json_data = json.dumps(data, separators=(',', ':'))
myPrint(json_data)
if db.checkRequestInCredentialTransaction(json_data) is None:
resp = ms.credentialRequest(data)
output.append(resp)
else:
myPrint("Skipping credential request", 11)
writeJsonFile(vidRequestId, output)
myPrint('Input VIDs: ' + str(len(vids)), 12)
myPrint('Output RequestIds: ' + str(len(output)), 12)
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action reprint: " + prstr, 11)
db.closeAll()
except:
db.closeAll()
prev_time, prstr = timeDiff(start_time)
myPrint("Total time taken by the script: " + prstr, 11)
formatted_lines = traceback.format_exc()
myPrint(formatted_lines, 13)
sys.exit(1)
prev_time, prstr = timeDiff(start_time)
myPrint("Total time taken by the script: " + prstr, 11)
return sys.exit(0)