def get_or_create_subscription():
conf = get_config()["google_pub_sub"]
project_id, topic_id = conf["project_id"], conf["topic_id"]
subscription_id = get_subs_name(conf["subscription"].get(
"type", "schedule-consumer"))
subscriber = pubsub_v1.SubscriberClient()
publisher = pubsub_v1.PublisherClient()
sub_path = subscriber.subscription_path(project_id, subscription_id)
topic_path = publisher.topic_path(project_id, topic_id)
try:
subscriber.create_subscription(
request={
"name":
sub_path,
"topic":
topic_path,
"message_retention_duration":
Duration(seconds=conf["subscription"].get(
"message_retention_duration", 86400)),
"ack_deadline_seconds":
conf["subscription"].get("ack_deadline_seconds", 300),
"filter":
f'attributes.mac = "{get_mac()}"'
})
logging.info(f"{sub_path} created")
except AlreadyExists:
logging.info(f"{sub_path} already exists")
return sub_path
return sub_path
def main():
config = get_config(service_name='temperature_sensor', log=log)
log.info("header: datetime, device_uuid, device_tags, temperature_celcius")
action_func = action_func_factory(log=log, config=config)
batch_func = batch_send_factory(log=log,
config=config,
batch_limit=10,
verify_ssl_certificate=False)
intervaled_ma(log=log,
action_func=action_func,
batch_func=batch_func,
min_seconds_between_actions=10)
def main():
log.info("log stats started")
config = get_config(service_name='stat_reporter', log=log)
action_func = action_func_factory(log=log, config=config)
batch_func = batch_send_factory(log=log,
config=config,
batch_limit=1,
verify_ssl_certificate=False)
intervaled_ma(log=log,
action_func=action_func,
batch_func=batch_func,
min_seconds_between_actions=100)
def __init__(self):
self.jobstores = {
'default': RedisJobStore(),
'redis': RedisJobStore()
}
self.executors = {
'default': ProcessPoolExecutor(max_workers=get_config()["scheduler"].get("process_pool_max_workers", 20))
}
self.job_defaults = {
'coalesce': True,
'max_instances': 1
}
self.scheduler = BackgroundScheduler(jobstores=self.jobstores,
executors=self.executors,
job_defaults=self.job_defaults)
def config_app(p_app):
"""
配置app参数
:param p_app: Flask实例
:return:
"""
config = get_config('aliyun_mysql')
db_type = config['db_type']
host = config['host']
port = config['port']
user = config['user']
password = config['password']
p_app.config[
'SQLALCHEMY_DATABASE_URI'] = f'{db_type}+pymysql://{user}:{password}@{host}:{port}/watero'
p_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False # 不追踪数据库变化
p_app.config['SQLALCHEMY_ECHO'] = False # 不打印原始SQL语句
def __init__(self):
import adafruit_rfm9x
import board
import busio
from digitalio import DigitalInOut
cs = DigitalInOut(board.CE1)
reset = DigitalInOut(board.D25)
spi = busio.SPI(board.SCK, MOSI=board.MOSI, MISO=board.MISO)
self.__config = get_config()["lora"]
self.__actions = self.__config["actions"]
self.rfm9x = adafruit_rfm9x.RFM9x(spi, cs, reset,
self.__config["frecuency"])
self.serve_action = self.__actions["serve"]
self.success_led = 13
self.fail_led = 26
self.warn_led = 19
self.__setup_leds()
import numpy as np
import json
if __name__ == "__main__":
args = argparser()
print("saving file to {}".format(args.prefix))
# create workspace
workspace_dir = "experiments/{}".format(args.prefix)
if not os.path.isdir(workspace_dir):
os.makedirs(workspace_dir, exist_ok=True)
test_log = open("{}/test.log".format(workspace_dir), "w")
input_size, _, test_loader = get_config(args.data)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
random.seed(0)
np.random.seed(0)
encoder = GlobalEncoder(stride=args.encoder_stride,
input_size=input_size,
output_size=args.code_size)
# load encoder from checkpoint
encoder.load_state_dict(
torch.load(args.encoder_ckpt)["encoder_state_dict"])
encoder = encoder.to(args.device)
decoder = DeconvDecoder(input_size=encoder.output_size,
output_size=input_size)
import json
if __name__ == "__main__":
args = argparser()
print("saving file to {}".format(args.prefix))
# create workspace
workspace_dir = "experiments/{}".format(args.prefix)
if not os.path.isdir(workspace_dir):
os.makedirs(workspace_dir, exist_ok=True)
train_log = open("{}/train.log".format(workspace_dir), "w")
test_log = open("{}/test.log".format(workspace_dir), "w")
input_size, ndf, num_channels, train_loader, test_loader = get_config(args)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
random.seed(0)
np.random.seed(0)
encoder = GlobalEncoder(ndf=ndf,
num_channels=num_channels,
output_size=args.code_size,
input_size=input_size)
if args.global_dim:
DIM = GlobalDIM(encoder, type=args.mi_estimator)
else:
DIM = LocalDIM(encoder, type=args.mi_estimator)
def __init__(self, database, collection, where, json_file, export):
if where == 'local':
#local data
json_file = json_file
try:
client = MongoClient('localhost', 27017)
self.__db = client[database]
#create a new collection if it does not exist in database
if collection not in self.__db.list_collection_names():
self.__collection = self.__db[collection]
file_data = []
for line in open(json_file, 'r'):
file_data.append(loads(line))
self.__collection.insert_many(file_data)
# print the version of MongoDB server if connection successful
print("server version:", client.server_info()["version"])
except errors.ServerSelectionTimeoutError as err:
client = None
print("pymongo ERROR:", err)
self.__database = database
self.__collection = collection
if where == 'remote':
#connection configuration file
json_file = json_file
script_parent_dir = pathlib.Path(__file__).parents[1]
config_fn = script_parent_dir.joinpath(json_file)
config = get_config(config_fn)
connection_dict = {
'host': config['mongodb']['host'],
'port': int(config['mongodb']['port']),
'username': config['mongodb']['username'],
'password': config['mongodb']['password']
}
self.__database = database
self.__collection = collection
uri = "mongodb://%s:%s@%s:%s/%s" % (
connection_dict['username'], connection_dict['password'],
connection_dict['host'], connection_dict['port'], database)
try:
client = MongoClient(
uri, serverSelectionTimeoutMS=3000) # 3 second timeout
self.__db = client[self.__database]
# print the version of MongoDB server if connection successful
print("server version:", client.server_info()["version"])
except errors.ServerSelectionTimeoutError as err:
client = None
print("pymongo ERROR:", err)
if export == 'yes':
export_path = '../sna/data/es-tweets.hpai.json'
self.__query = {'$and': [{'lang': {'$in': SPAIN_LANGUAGES}},
{'$or': [{'place.country': 'Spain'},
{'user.location': {'$in': \
get_spain_places_regex()}}]}]}
print("Exporting cursor...")
cursor = self.search(self.__query)
file = open(export_path, "w")
for document in tqdm(cursor, total=470000):
file.write(dumps(document))
file.write("\n")
sys.exit("Bye!")
# Setup/Import ---------------------------------------------------------------- from fastapi import APIRouter, HTTPException from pydantic import BaseModel, ValidationError from typing import Dict, List, Union, Any from typing_extensions import Literal import os.path as path import mlflow import json from mlflow.tracking import MlflowClient from utils.get_config import get_config import time from fastapi.encoders import jsonable_encoder from starlette.responses import HTMLResponse, JSONResponse # Grab Config conf = get_config() # URI uri = conf['uri'] mlClient = MlflowClient(tracking_uri=uri) mlflow.set_tracking_uri(uri) # Experiment Name experiment_name = conf['experiment_name'] mlflow.set_experiment(experiment_name) experiment_id = mlflow.get_experiment_by_name(experiment_name).experiment_id router = APIRouter() # Process Endpoint ------------------------------------------------------------
