def get_token(self):
res = requests.post(
"https://aip.baidubce.com/oauth/2.0/token",
params={
"grant_type": "client_credentials",
"client_id": config.get("OCR", "APIKEY"),
"client_secret": config.get("OCR", "SecretKey"),
},
)
self.token = res.json()["access_token"]
print("Token:", self.token)
def save_jpeg(
path, image, now=None, title=None, subject=None, comment=None, keywords=None
):
if not path.endswith((".jpg", ".jpeg")):
path = path + ".jpg"
image = Image.fromarray(cv.cvtColor(image, cv.COLOR_BGR2RGB))
path = os.path.join(config.get("Path", "StorageDir"), path)
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname)
exif_dict = {ifd: {} for ifd in ("0th", "Exif", "GPS", "1st")}
exif_dict["0th"][piexif.ImageIFD.Artist] = "AutumnSun"
exif_dict["0th"][piexif.ImageIFD.Software] = "AutoDroid"
if now is None:
now = datetime.datetime.now()
exif_dict["Exif"][piexif.ExifIFD.DateTimeDigitized] = now.strftime(
"%Y:%m:%d %H:%M:%S"
)
exif_dict["Exif"][piexif.ExifIFD.SubSecTimeDigitized] = "{:.0f}".format(
now.microsecond
)
if title is not None:
exif_dict["0th"][piexif.ImageIFD.ImageDescription] = title.encode("UTF-8")
if subject is not None:
exif_dict["0th"][piexif.ImageIFD.XPSubject] = get_xp_info(subject)
if comment is not None:
exif_dict["0th"][piexif.ImageIFD.XPComment] = get_xp_info(comment)
if keywords is not None:
exif_dict["0th"][piexif.ImageIFD.XPKeywords] = get_xp_info(keywords)
exif_bytes = piexif.dump(exif_dict)
path = os.path.abspath(path)
image.save(path, "jpeg", exif=exif_bytes)
return path
def login_weibo(cls):
state = request.args.get('state', '')
weibo_config = config.get('oauth')['weibo']
login_url = "%s?client_id=%s&response_type=code&state=%s&redirect_uri=%s" % (
weibo_config['login_url'], weibo_config['app_key'], state,
weibo_config['redirect_url'])
return redirect(login_url, code=301)
def callback_weibo(cls):
code = request.args.get('code', None)
state = request.args.get('state', '') # 用于给webxoss验证用参数
if code == None:
return jsonify({'result': -1, 'msg': 'Code Error'})
weibo_config = config.get('oauth')['weibo']
# 根据code获取uid信息
get_params = {
'client_id': weibo_config['app_key'],
'client_secret': weibo_config['app_secret'],
'redirect_uri': weibo_config['redirect_url'],
'grant_type': 'authorization_code',
'code': code,
}
res = requests.post(weibo_config['access_token_request_url'],
params=get_params)
res_obj = json.loads(res.text)
weibo_uid = res_obj['uid']
# 根据uid在数据库中查询用户
user = UserWeibo.login_by_uid(weibo_uid)
# 回传一个Redirect指令
return CallbackController.redirect_linker(user.token,
User._REFRESH_TIME, state)
def __init__(self, map_name, extra_property=None):
self.hwnd = get_window_hwnd(config.get(self.section, "WindowTitle"))
self.adb_name = config.get(self.section, "AdbName")
set_logging_dir(os.path.join(self.section, "logs"))
self.map_name = map_name
self.data = load_map(self.map_name, self.section, extra_property)
self.scene_history = deque(maxlen=50)
self.scene_history_count = defaultdict(zero)
self.last_change = time.time()
self.screen = None
self.actions_done = False
self.scenes = self.data["Scenes"]
self.resources = self.data["Resources"]
self.last_manual = time.time()
self.stop = False
self.resource_pos_buffer = {}
self.call_once_history = set()
def load_map(name, section, extra_property):
path = os.path.join(config.get(section, "ResourcesFolder"), "maps", name + ".conf")
data = hocon.load(path)
if extra_property:
new_args = hocon.loads("\n".join(extra_property))
data = new_args.with_fallback(data)
for key, item in data["Resources"].items():
update_resource(item, section, key)
if "Anchors" in data:
for val in data["Anchors"].values():
update_resource(val, section)
return data
def load_image(resource, section):
if "ImageData" not in resource:
img_path = os.path.join(
config.get(section, "ResourcesFolder"), "Resources", resource["Image"]
)
resource["ImageData"] = cv_imread(img_path)
if "Size" in resource:
resource["ImageData"] = cv.resize(
resource["ImageData"], tuple(resource["Size"]), cv.INTER_CUBIC
)
else:
h, w = resource["ImageData"].shape[:2]
resource["Size"] = [w, h]
return resource["ImageData"]
def main(section, rows):
window_title = config.get(section, "WindowTitle")
hwnd = get_window_hwnd(window_title)
if not hwnd:
print("No Such Window")
return
x0, y0 = 100, 200
dx, dy = int(800 / 6), int(430 / 3)
w, h = 80, 80
for i in range(7):
for j in range(rows):
x, y = (x0 + dx * i, y0 + dy * j)
rand_click(hwnd, (x, y, x + w, y + h))
time.sleep(0.3)
def load_resources(section):
with open(config.get(section, "Resources"), "r", -1, "UTF-8") as fl:
items = yaml.load(fl)
for key, resource in items.items():
update_resource(resource, section, key)
return items
def load_scenes(section):
with open(config.get(section, "Scenes"), "r", -1, "UTF-8") as fl:
items = yaml.load(fl)
return items
import argparse
import logging
import os
from datetime import datetime
import cv2.cv2 as cv
from simulator import image_tools, win32_tools
from config_loader import config
logging.basicConfig(level="DEBUG")
basedir = config.get("Path", "StorageDir")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("title", default="FGO1", help="Title of Simulator")
parser.add_argument("save_dir",
nargs="?",
default=None,
help="Dir to save shot.")
args = parser.parse_args()
if args.save_dir is None:
args.save_dir = args.title + "/shots"
hwnd = win32_tools.get_window_hwnd(args.title)
img = win32_tools.get_window_shot(hwnd)
path = os.path.join(basedir, args.save_dir,
"Shot-{:%Y-%m-%d_%H%M%S}.jpg".format(datetime.now()))
window_name = "Screen Shot Preview"
import os
import time
import re
import numpy as np
import cv2.cv2 as cv
from adb_shell.adb_device import AdbDeviceTcp, AdbDeviceUsb
from config_loader import config
ADB_PATH = config.get("Path", "ADB")
devices = {}
def init_device(name):
lines = os.popen("{} devices -l".format(ADB_PATH)).read()
mds = re.findall(r"(?m)^([\d\.]+):(\d+).+?model:(\w+) ", lines)
mds = {name: (ip, int(port)) for ip, port, name in mds}
ip, port = mds[name]
device = AdbDeviceTcp(ip, port)
device.connect()
return device
def get_screencap(info):
if info not in devices:
devices[info] = init_device(info)
device = devices[info]
data = device.shell("screencap -p", decode=False)
# 修复换行符转换导致的数据异常
if data.startswith(b'\x89PNG\r\r\n'):
data = data.replace(b'\r\n', b'\n')
def __init__(
self,
train: bool = True,
val: bool = False,
transform: Optional[object] = None,
target_transform: Optional[object] = None,
debug_with_few_samples: bool = False,
create_standardization_transform: bool = False,
) -> None:
if train and val:
print('"train" and "val" cannot be true at the same time!')
raise ValueError
min_year = config.getint('DataOptions', 'min_year')
max_year = config.getint('DataOptions', 'max_year')
max_train_year = config.getint('DataOptions', 'max_train_year')
max_val_year = config.getint('DataOptions', 'max_val_year')
remo_input_dir = config.get('Paths', 'remo_input')
remo_target_dir = config.get('Paths', 'remo_target')
elev_file = config.get('Paths', 'elevation')
input_var = config.get('DataOptions', 'input_variable')
target_var = config.get('DataOptions', 'target_variable')
aux_base_path = config.get('Paths', 'aux_base_path')
# The filter removes empty strings in the resulting list, which occur when there are no aux_variables specified
aux_vars = list(
filter(None,
config.get('DataOptions', 'aux_variables').split(',')))
aux_vars = [join(aux_base_path, p) for p in aux_vars]
if train:
self.years = list(range(min_year, max_train_year + 1))
print('Train years', self.years)
self.mode = 'train'
elif val:
self.years = list(range(max_train_year + 1, max_val_year + 1))
print('Validation years', self.years)
self.mode = 'val'
else:
self.years = list(range(max_val_year + 1, max_year + 1))
print('Test years', self.years)
self.mode = 'test'
self.transform = transform
self.target_transform = target_transform
# For testing purposes
# self.years = [2000]
self.dataset = RemoSuperRes(
remo_input_dir,
remo_target_dir,
self.years,
elev_file,
input_var=input_var,
target_var=target_var,
aux_features=aux_vars,
)
# This name is misleading - it does not necessarily create test data
(
self.X,
self.aux,
self.elev_arr,
self.Y,
self.lats,
self.lons,
self.times,
) = self.dataset.make_test()
if debug_with_few_samples:
num_debug_samples = 16
print(f'DEBUG: Using only {num_debug_samples} samples')
self.X = self.X[:num_debug_samples]
if self.aux is not None:
self.aux = self.aux[:num_debug_samples]
self.Y = self.Y[:num_debug_samples]
self.times = self.times[:num_debug_samples]
# Convert E-OBS temperature from Celsius to Kelvin
if target_var == 'tg':
self.Y += 272.15
if create_standardization_transform:
self.standardize_transform = self.calculate_standardization_transform(
)
if self.transform is not None:
self.transform = Compose(
[self.transform, self.standardize_transform])
else:
self.transform = self.standardize_transform
def export(lr: float, model_name: str, save_dir: str):
models_available = {
'globalnet': GlobalNet,
'localnet': LocalNet,
'convmos': ConvMOS,
}
model = models_available[model_name]()
device = 'cpu'
if torch.cuda.is_available():
device = 'cuda'
model = model.to(device=device)
# E-OBS only provides observational data for land so we need to use a mask to avoid fitting on the sea
land_mask_np = np.load('remo_eobs_land_mask.npy')
# Convert booleans to 1 and 0, and convert numpy array to torch Tensor
land_mask = torch.from_numpy(1 * land_mask_np).to(device)
loss_fn = partial(masked_mse_loss, mask=land_mask)
optimizer = Adam(model.parameters(), lr=lr)
trainer = create_supervised_trainer(model,
optimizer,
loss_fn,
device=device)
to_save = {'model': model, 'optimizer': optimizer, 'trainer': trainer}
checkpoint_files = glob(join(save_dir, 'best_checkpoint_*.pt'))
if len(checkpoint_files) > 0:
# Parse something like:
# /scratch/scratch_remo/APRL-rr-11-11-sdnext-loglonet-prec-ger11-maskedloss-7/best_checkpoint_194_val_loss=-11.8250.pt
# Sorry
epoch_to_score = {
int(c.split(sep)[-1].split('_')[2]):
float(c.split(sep)[-1].split('=')[-1][:-3])
for c in checkpoint_files
}
print(epoch_to_score)
best_epoch = max(epoch_to_score, key=epoch_to_score.get)
best_checkpoint_file = next(
cf for cf in checkpoint_files
if int(cf.split(sep)[-1].split('_')[2]) == best_epoch)
print('Loading best checkpoint', best_checkpoint_file)
checkpoint = torch.load(best_checkpoint_file, map_location=device)
Checkpoint.load_objects(to_load=to_save, checkpoint=checkpoint)
else:
print(
'ERROR: cannot find any files matching',
join(save_dir, 'best_checkpoint_*.pt'),
)
return
# This uses all aux variables, the temperature/precipitation, and elevation
input_depth = (len(
list(
filter(None,
config.get('DataOptions', 'aux_variables').split(',')))) +
2)
input_width = config.getint('NN', 'input_width')
input_height = config.getint('NN', 'input_height')
dummy_input = torch.randn(1,
input_depth,
input_width,
input_height,
device=device)
torch.onnx.export(
model,
dummy_input,
join(save_dir, 'convmos.onnx'),
verbose=True,
input_names=['input'],
output_names=['output'],
)
input_names=['input'],
output_names=['output'],
)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument(
'--lr',
type=float,
default=config.getfloat('NN', 'learning_rate'),
help='learning rate (default: 0.001)',
)
parser.add_argument(
'--model',
type=str,
default=config.get('NN', 'model'),
help='Which model to use',
)
parser.add_argument(
"--save_dir",
type=str,
default=join(config.get('NN', 'scratch'),
config.get('SD', 'model_name')),
help="save directory for checkpoint output",
)
args = parser.parse_args()
export(args.lr, args.model, args.save_dir)
help='Max cell distance to consider (Only relevant in non_local mode)',
)
parser.add_argument(
'--n_components',
type=float,
default=config.getfloat('NN', 'n_components'),
help=
'Number of components to retain for PCA (Only relevant in non_local mode)',
)
parser.add_argument(
'--n_param_sets',
type=int,
default=config.getint('NN', 'n_param_sets'),
help='Number of parameter sets tried per cell in the hp search',
)
parser.add_argument("--model", type=str, default=config.get('NN', 'model'))
parser.add_argument(
"--save_dir",
type=str,
default=join(config.get('NN', 'scratch'),
config.get('SD', 'model_name')),
)
args = parser.parse_args()
print(args)
run(
model_name=args.model,
num_jobs=args.num_jobs,
n_components=args.n_components,
