def __init_json_data(self):
if not os.path.exists(self.__node_pos_path):
util.write_json_file(file_path=self.__node_pos_path, data={})
logging.info("创建文件 - %s" % self.__node_pos_path)
else:
self.node_pos_data = util.read_json_file(
file_path=self.__node_pos_path)
def update(self, updater: Updater):
logging.info("update-service update")
self.data_handler.update()
if self.last_update["vaccinationsLastUpdated"].split(
)[0] != self.data_handler.update_info["vaccinationsLastUpdated"].split(
)[0]:
logging.info("new Data available")
self.last_update = self.data_handler.update_info.copy()
util.write_json_file(self.last_update, "last-update.json")
_, msg = self.message_generator.gen_text("numbers")
msg = strings["auto-update-text"].format(msg)
update_notice = self.__get_update_notice()
for chat_id in self.subscriptions.keys():
if "zahlen" in self.subscriptions[chat_id]:
try:
updater.bot.send_message(chat_id,
msg,
parse_mode=ParseMode.HTML)
if update_notice is not None:
updater.bot.send_message(chat_id,
update_notice,
parse_mode=ParseMode.HTML)
logging.info("sent update to {}".format(chat_id))
except Exception as e:
logging.error(e)
def main_csv_reader(args):
path_to_coffee = args.path_to_coffee
path_to_matched = args.matched_json
all_people_list = flat_list(list(read_csv_file(path_to_coffee)))
matched_in_this_session = []
error = False
if path_to_matched:
try:
matched_people_json = read_json_file(path_to_matched)
tuple_list = create_tuple_list(all_people_list,
matched_people_json)
sorted_people_list = sort_tuple_list(tuple_list)
except:
raise ('Only use the program generated matched_people.json file')
else:
write_json_file()
matched_people_json = read_json_file('matched_people.json')
sorted_people_list = all_people_list
unmatched_people = []
for person in sorted_people_list:
if person not in matched_in_this_session:
individual_match_list = invidual_preproc(person, all_people_list,
matched_people_json,
matched_in_this_session)
if individual_match_list:
matched_pair = coffee_roulette(person, individual_match_list)
if matched_pair is not None:
for person in matched_pair:
matched_in_this_session.append(person)
else:
error = True
break
else:
unmatched_people.append(person)
else:
pass
if error is False:
create_today_matched(matched_in_this_session)
if unmatched_people:
create_today_unmatched(unmatched_people)
updated_json = update_current_json(matched_people_json,
matched_in_this_session)
summary = "\n{} Matches".format(date.today())
summary = create_matched_people_string(matched_in_this_session,
summary)
summary_messsage, alone = make_summary(matched_in_this_session,
unmatched_people, summary, "")
summary += alone
write_json_file(updated_json)
write_txt_file(summary)
print(summary_messsage)
def __init_robots(self):
value_count = 0
if self.__start_uid > 0:
uid_list = list()
for uid in [
uid for uid in range(self.__start_uid, self.__start_uid +
self.__robots_num)
]:
player = Robot(sid=self.__sid, uid=uid, ksid=self.__sid)
if player.has_login:
self.robot_object_list.append(player)
uid_list.append(uid)
value_count += 1
else:
logging.error("%s login failed" % uid)
else:
uid_list = []
robot_count = 0
while robot_count < self.__robots_num:
player = Robot(sid=self.__sid, uid=0, ksid=self.__sid)
if player.has_login:
self.robot_object_list.append(player)
uid_list.append(player.uid)
value_count += 1
else:
logging.error("create account failed")
robot_count += 1
logging.info("login success num: %s ; total num : %s" %
(value_count, self.__robots_num))
try:
self.__robots_data = read_json_file(
file_path=config.robots_json_data_path)
except FileNotFoundError:
self.__robots_data[str(config.okc_environment)] = {
str(self.__sid): uid_list
}
logging.info("robots_data : %s" % self.__robots_data)
try:
old_uid = set(self.__robots_data[str(config.okc_environment)][str(
self.__sid)])
new_uid = set(uid_list)
self.__robots_data[str(config.okc_environment)][str(
self.__sid)] = list(old_uid | new_uid)
except KeyError:
import traceback
logging.error(traceback.print_exc())
last_load = {
"env": config.okc_environment,
"kingdom": self.__sid,
"sid": self.__sid,
"uid": uid_list
}
self.__robots_data["last_load"] = last_load
write_json_file(file_path=config.robots_json_data_path,
data=self.__robots_data)
def __save_robots(self):
env_path = util.get_ini_data(ini_path=config.conf_path,
section="path",
section_item="okc_environment")
last_load = util.read_json_file(env_path)
if self.robots:
robot_uid_list = []
for robot in self.robots:
robot_uid_list.append(robot.uid)
last_load["uid"] = robot_uid_list
# data = {"sid": self.sid, "player_list": robot_uid_list}
util.write_json_file(file_path=env_path, data=last_load)
def __get_douyu_rooms(self):
# 获取斗鱼平台直播的所有房间
self.browser.get(config.douyu.base_url + "/directory")
# util.write_data(write_path=config.douyu.directory_path, data=self.browser.page_source)
soup_menu = bs4.BeautifulSoup(self.browser.page_source, "html.parser")
for data in soup_menu.select("script"):
data = str(data)
if "DATA" in data:
data = data.split("var $DATA =")[-1].split(";")[0]
self.directory_games = json.loads(data)
print(self.directory_games)
# 所有游戏类型,包含游戏名称列表
category_list = self.directory_games["firstCategory"]
# 游戏名称 eg.LOL
game_relative_path_list = []
game_url_list = []
for category in category_list:
for relative in category["secondCategory"]:
game_relative_path_list.append(relative["cate2Url"])
for game_relative_path in game_relative_path_list:
url = config.douyu.base_url + game_relative_path
game_url_list.append(url)
util.write_json_file(os.getcwd() + "\\game_url.json", data=game_url_list)
# 获取房间号
# TODO 获取房间属性 -- 关注量，名称等等，用于实现筛选功能，过滤官方直播间。可用class 描述属性实现
for game_url in game_url_list:
time.sleep(config.request_wait_time)
game_url_result = requests.get(game_url)
if game_url_result.status_code == 200:
logging.info("%s 访问成功" % game_url)
else:
logging.error("%s 访问失败" % game_url)
continue
soup_game = bs4.BeautifulSoup(game_url_result.text, "html.parser")
game_room_list = soup_game.select(".layout-Cover-item")
for game_room in game_room_list:
self.douyu_rooms.append("%s/%s" % (config.douyu.base_url, game_room.select("a")[0].get("href")))
logging.info("斗鱼旗下所有直播间的数量为 : %s" % len(self.douyu_rooms))
util.write_json_file(os.getcwd() + "\\room_url.json", data=self.douyu_rooms)
def write_geojson(args, source, fc, fn, archive, updated_stations):
fc.properties["processed"] = int(datetime.utcnow().timestamp())
fc.properties["origin_member"] = pathlib.PurePath(fn).name
if archive:
fc.properties["origin_archive"] = pathlib.PurePath(archive).name
station_id = fc.properties["station_id"]
if args.station and args.station != station_id:
return
fc.properties["fmt"] = config.FORMAT_VERSION
logging.debug(
f"output samples retained: {len(fc.features)}, station id={station_id}"
)
updated_stations.append((station_id, fc.properties))
cc = station_id[:2]
subdir = station_id[2:5]
syn_time = datetime.utcfromtimestamp(fc.properties["syn_timestamp"]).replace(
tzinfo=pytz.utc
)
day = syn_time.strftime("%Y%m%d")
year = syn_time.strftime("%Y")
month = syn_time.strftime("%m")
time = syn_time.strftime("%H%M%S")
dest = (
f"{args.destdir}/{source}/{cc}/{subdir}/"
f"{year}/{month}/{station_id}_{day}_{time}.geojson.br"
)
ref = f"{source}/{cc}/{subdir}/" f"{year}/{month}/{station_id}_{day}_{time}.geojson"
path = pathlib.Path(dest).parent.absolute()
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
if not fc.is_valid:
logging.error(f"--- invalid GeoJSON! {fc.errors()}")
raise ValueError("invalid GeoJSON")
util.write_json_file(fc, dest, useBrotli=True, asGeojson=True)
fc.properties["path"] = ref
if args.dump_geojson:
pprint(fc)
return True
def join_all():
requests = read_compressed_json_file(
'../data/requests.json.gzip')["objects"]
messages = read_compressed_json_file(
'../data/messages.json.gzip')["objects"]
data = []
for r in requests:
request_url = f"https://fragdenstaat.de/api/v1/request/{r['id']}/"
print(request_url)
new_data = [m for m in messages if m["request"] == request_url]
data.append({"request": r, "messages": new_data})
write_json_file('../data/full_messages.json', data)
def init_cookies(self, platform):
xpath_login = '******'
xpath_player = '//*[@id="js-header"]/div/div/div[3]/div[7]/div/a/span/div/div/img'
self.browser.get(url=self.init_url)
self.browser.maximize_window()
time.sleep(2)
self.log_text_send.emit(self.browser.current_url)
if util.find_xpath_in_xml(self.browser, xpath_login):
self.browser.find_element_by_xpath(xpath=xpath_login).click()
if util.find_xpath_in_xml(self.browser, xpath_player):
util.write_json_file(
file_path=config.cookies_path,
data={platform: self.browser.get_cookies()[0]})
def join_successful_messages():
requests = read_compressed_json_file(
'../data/requests.json.gzip')["objects"]
messages = read_compressed_json_file(
'../data/messages.json.gzip')["objects"]
data = []
for r in requests:
if r["resolution"] in ["successful", "partially_successful"]:
request_url = f"https://fragdenstaat.de/api/v1/request/{r['id']}/"
print(request_url)
new_data = [
m for m in messages
if m["is_response"] and m["request"] == request_url
]
data.extend(new_data)
write_json_file('../data/suc_msg.json', data)
def click_by_node_name(self,
view,
node_name,
can_change=False,
result_view="") -> bool:
if can_change:
node_pos = self.get_pos(node_name)
elif view in self.ui_pos.keys():
if node_name in self.ui_pos[view].keys():
node_pos = tuple(self.ui_pos[view][node_name])
else:
node_pos = self.get_pos(node_name)
self.ui_pos[view][node_name] = list(node_pos)
util.write_json_file(file_path=self.ui_pos_path,
data=self.ui_pos)
else:
node_pos = self.get_pos(node_name)
self.ui_pos[view] = {node_name: list(node_pos)}
util.write_json_file(file_path=self.ui_pos_path, data=self.ui_pos)
logging.info("%s 的坐标为 : %s" % (node_name, node_pos))
if node_pos != ():
self.poco.click(pos=node_pos)
api.sleep(self.click_wait_time)
self.update_ui_node()
if result_view == "":
return True
if self.find_ui_node(ui_node_name=result_view):
return True
else:
logging.error("点击%s后，没有打开%s" % (node_name, result_view))
return False
else:
self.phone.get_screen_shot(node_name, self.default_save_path)
logging.error("点击失败，没找到UI坐标")
return False
def update_geojson_summary(args, stations, updated_stations, summary):
stations_with_ascents = {}
# unroll into dicts for quick access
if "features" in summary:
for feature in summary.features:
a = feature.properties["ascents"]
if len(a):
st_id = feature.properties["station_id"]
stations_with_ascents[st_id] = feature
# remove entries from ascents which have a syn_timestamp less than cutoff_ts
cutoff_ts = util.now() - args.max_age * 24 * 3600
# now walk the updates
for station, asc in updated_stations:
if station in stations_with_ascents:
# we already have ascents from this station.
# append, sort by synoptic time and de-duplicate
oldlist = stations_with_ascents[station]["properties"]["ascents"]
oldlist.append(asc)
pruned = [x for x in oldlist if x["syn_timestamp"] > cutoff_ts]
logging.debug(f"pruning {station}: {len(oldlist)} -> {len(pruned)}")
newlist = sorted(pruned, key=itemgetter("syn_timestamp"), reverse=True)
# https://stackoverflow.com/questions/9427163/remove-duplicate-dict-in-list-in-python
seen = set()
dedup = []
for d in newlist:
# keep an ascent of each source, even if same synop time
t = str(d["syn_timestamp"]) + d["source"]
if t not in seen:
seen.add(t)
dedup.append(d)
stations_with_ascents[station]["properties"]["ascents"] = dedup
# fixup the name if it was added to station_list.json:
ident = stations_with_ascents[station]["properties"]["name"]
if ident in stations:
# using WMO id as name. Probably mobile. Replace by string name.
stations_with_ascents[station]["properties"]["name"] = stations[ident][
"name"
]
# overwrite the station coords by the coords of the last ascent
# to properly handle mobile stations
if asc["id_type"] == "mobile":
logging.debug(
f"fix coords {station} -> {asc['lon']} {asc['lat']} {asc['elevation']}"
)
properties = stations_with_ascents[station]["properties"]
stations_with_ascents[station] = geojson.Feature(
geometry=geojson.Point(
(
round(asc["lon"], 6),
round(asc["lat"], 6),
round(asc["elevation"], 1),
)
),
properties=properties,
)
else:
# station appears with first-time ascent
properties = {}
properties["ascents"] = [asc]
if station in stations:
st = stations[station]
coords = (st["lon"], st["lat"], st["elevation"])
properties["name"] = st["name"]
properties["station_id"] = station
properties["id_type"] = "wmo"
else:
# unlisted station: anonymous + mobile
# take coords and station_id as name from ascent
coords = (asc["lon"], asc["lat"], asc["elevation"])
properties["name"] = asc["station_id"]
if re.match(r"^\d{5}$", station):
# WMO id syntax, but not in station_list
# hence an unregistered but fixed station
properties["id_type"] = "unregistered"
else:
# looks like weather ship
properties["id_type"] = "mobile"
stations_with_ascents[station] = geojson.Feature(
geometry=geojson.Point(coords), properties=properties
)
# create GeoJSON summary
ns = na = 0
fc = geojson.FeatureCollection([])
fc.properties = {
"fmt": config.FORMAT_VERSION,
"generated": int(util.now()),
"max_age": args.max_age * 24 * 3600,
}
for _st, f in stations_with_ascents.items():
sid, stype = slimdown(f)
f.properties["station_id"] = sid
f.properties["id_type"] = stype
ns += 1
na += len(f.properties["ascents"])
fc.features.append(f)
logging.debug(f"summary {args.summary}: {ns} active stations, {na} ascents")
useBrotli = args.summary.endswith(".br")
util.write_json_file(fc, args.summary, useBrotli=useBrotli, asGeojson=True)
def initialize_stations(txt_fn, json_fn):
US_STATES = [
"AK",
"AL",
"AR",
"AZ",
"CA",
"CO",
"CT",
"DE",
"FL",
"GA",
"HI",
"IA",
"ID",
"IL",
"IN",
"KS",
"LA",
"MA",
"MD",
"ME",
"MI",
"MN",
"MO",
"MS",
"MT",
"NC",
"ND",
"NE",
"NH",
"NJ",
"NM",
"NV",
"NY",
"OH",
"OK",
"OR",
"PA",
"RI",
"SC",
"SD",
"TN",
"TX",
"UT",
"VA",
"VT",
"WA",
"WI",
"WV",
"WY",
]
stationdict = {}
with open(txt_fn, "r") as csvfile:
stndata = csv.reader(csvfile, delimiter="\t")
for row in stndata:
m = re.match(
r"(?P<stn_wmoid>^\w+)\s+(?P<stn_lat>\S+)\s+(?P<stn_lon>\S+)\s+(?P<stn_altitude>\S+)(?P<stn_name>\D+)",
row[0],
)
fields = m.groupdict()
stn_wmoid = fields["stn_wmoid"][6:]
stn_name = fields["stn_name"].strip()
if re.match(r"^[a-zA-Z]{2}\s",
stn_name) and stn_name[:2] in US_STATES:
stn_name = stn_name[2:].strip().title() + ", " + stn_name[:2]
else:
stn_name = stn_name.title()
stn_name = fields["stn_name"].strip().title()
stn_lat = float(fields["stn_lat"])
stn_lon = float(fields["stn_lon"])
stn_altitude = float(fields["stn_altitude"])
if stn_altitude > -998.8:
stationdict[stn_wmoid] = {
"name": stn_name,
"lat": stn_lat,
"lon": stn_lon,
"elevation": stn_altitude,
}
util.write_json_file(stationdict, json_fn)
def main():
parser = argparse.ArgumentParser(
description="rebuild radiosonde summary.json",
add_help=True,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("-v", "--verbose", action="store_true", default=False)
parser.add_argument(
"--station-json",
action="store",
default=config.STATION_LIST,
help="path to write the station_list.json file",
)
parser.add_argument(
"--station-text",
action="store",
default=config.STATION_TXT,
help="path to the source text file to generate the station_list.json",
)
parser.add_argument(
"--summary",
action="store",
default=config.WWW_DIR + config.DATA_DIR + config.SUMMARY,
help="path of brotli-compressed summary.geojson.br",
)
parser.add_argument(
"--dirs",
nargs="+",
type=str,
default=[config.MADIS_DATA, config.GISC_DATA],
help="directories to scan for detail files (*.geojson.br)",
)
parser.add_argument(
"--max-age",
action="store",
type=int,
default=config.MAX_DAYS_IN_SUMMARY,
help="number of days of history to keep in summary",
)
parser.add_argument("--tmpdir", action="store", default=None)
args = parser.parse_args()
if args.tmpdir:
config.tmpdir = args.tmpdir
level = logging.WARNING
if args.verbose:
level = logging.DEBUG
logging.basicConfig(level=level)
os.umask(0o22)
if not os.path.exists(args.station_text):
logging.error(f"the {args.station_text} does not exist")
sys.exit(1)
for d in args.dirs:
if not os.path.exists(d):
logging.error(f"the directory {d} does not exist")
sys.exit(1)
try:
with pidfile.Pidfile(config.LOCKFILE,
log=logging.debug,
warn=logging.debug):
cutoff_ts = util.now() - args.max_age * 24 * 3600
update_station_list(args.station_text, args.station_json)
global station_list
station_list = json.loads(
util.read_file(args.station_json).decode())
ntotal = 0
for d in args.dirs:
nf, nu, nc = walkt_tree(d, pathlib.Path(d), "*.geojson.br",
cutoff_ts)
ntotal = ntotal + nf
fixup_flights(flights)
fc = geojson.FeatureCollection([])
fc.properties = {
"fmt": config.FORMAT_VERSION,
"generated": int(util.now()),
"max_age": config.MAX_DAYS_IN_SUMMARY * 24 * 3600,
}
for _st, f in flights.items():
fc.features.append(f)
util.write_json_file(fc,
args.summary,
useBrotli=True,
asGeojson=True)
for l in txtfrag:
print(l, file=sys.stderr)
except pidfile.ProcessRunningException:
logging.warning(f"the pid file {config.LOCKFILE}is in use, exiting.")
return -1
def okc_robot_start():
print(
"welcome to use okc robot.\n\nthe environment : 0-test ,1-dev ,2-online test ,3-online ."
.title())
print(
"no input or input error Use the environment where you last logged in."
.title())
print("If this is your first login, use test environment.\n".title())
input_env = input("choose environment :\n".title())
sid = input("enter sid:\n".title())
start_uid = input("start uid:\n".title())
robot_num = input("robot num:\n".title())
env_path = util.get_ini_data(ini_path=config.conf_path,
section="path",
section_item="okc_environment")
def get_help():
help_info = "-" * 150 + "\n"
for command_name in robots.__class__.__dict__.keys():
if not re.match(r"__", command_name):
help_info += "Command : %s Params : %s" % (
command_name + " " * (30 - len(command_name)) + "·" * 10,
robots.__getattribute__(command_name).__doc__)
help_info += "\n" + "-" * 150 + "\n"
return help_info
try:
input_env = int(input_env)
if input_env == 0:
config.okc_environment = config.OkcEnvironment.Test.value
elif input_env == 1:
config.okc_environment = config.OkcEnvironment.Dev.value
elif input_env == 2:
config.okc_environment = config.OkcEnvironment.OnlineTest.value
elif input_env == 3:
config.okc_environment = config.OkcEnvironment.Online.value
else:
config.okc_environment = config.OkcEnvironment.Test.value
logging.info("environment : %s " % config.ENV[config.okc_environment])
sid = int(sid)
start_uid = int(start_uid)
player_num = int(robot_num)
data = {
"env": config.okc_environment,
"sid": sid,
"uid": list(range(start_uid, start_uid + player_num))
}
util.write_json_file(env_path, data=data)
robots = AnalysisCommand(sid=sid,
start_uid=start_uid,
robot_num=player_num)
except ValueError:
last_load = util.read_json_file(env_path)
env = int(last_load["env"])
config.okc_environment = env
logging.info("environment : %s " % config.ENV[config.okc_environment])
print("-" * 150)
sid = int(last_load["sid"])
start_uid = last_load["uid"][0]
player_num = len(last_load["uid"])
if start_uid == -1:
robots = AnalysisCommand()
else:
robots = AnalysisCommand(sid=sid,
start_uid=start_uid,
robot_num=player_num)
if not robots.robots:
print("Robot Init Failed")
return
while True:
command = input("enter command : \n".title())
cmd = re.match(r"(\w+)(:?\s+(.*))", command)
if not cmd:
print("Input Error.\n")
print(get_help())
continue
try:
cmd_name, params = cmd.group(1, 2)
except ValueError:
print("Input Params Error.\n")
print(get_help())
continue
cmd_list = robots.__dir__()
if cmd_name in cmd_list:
cmd_value = getattr(robots, cmd_name)
try:
cmd_value(params)
except:
logging.error(traceback.print_exc())
else:
print("command -> '%s' isn't in cmd list.\n" % cmd_name)
print(get_help())
continue
def __akzeptieren(self, update: Update, context: CallbackContext):
"""Callback Methode für /akzeptieren - Fügt Nutzer zur DSGVO-Whitelist hinzu"""
if update.effective_chat.id not in self.whitelist:
self.whitelist.append(update.effective_chat.id)
util.write_json_file({"list": self.whitelist}, "whitelist.json")
self.__start(update, context)
def __write_json(self):
try:
util.write_json_file(self.subscriptions, "subscribers.json")
except Exception as e:
logging.error(e)
