def get_devicesettings_value(self, key: str, default_value: object = None):
logger.debug2("Fetching devicemappings of {}".format(self._id))
try:
devicemappings: Optional[
dict] = self._mapping_manager.get_devicemappings_of(self._id)
except (EOFError, FileNotFoundError) as e:
logger.warning(
"Failed fetching devicemappings in worker {} with description: {}. Stopping worker"
.format(str(self._id), str(e)))
return None
if devicemappings is None:
return default_value
return devicemappings.get("settings", {}).get(key, default_value)
def process_data(self, received_timestamp, data, origin):
global application_args
if origin not in self.__mitm_mapper.playerstats:
logger.warning("Not processing data of {} since origin is unknown",
str(origin))
return
type = data.get("type", None)
if type:
if type == 106:
# process GetMapObject
logger.success(
"Processing GMO received from {}. Received at {}",
str(origin),
str(datetime.fromtimestamp(received_timestamp)))
if application_args.weather:
self._db_wrapper.submit_weather_map_proto(
origin, data["payload"], received_timestamp)
self._db_wrapper.submit_pokestops_map_proto(
origin, data["payload"])
self._db_wrapper.submit_gyms_map_proto(origin, data["payload"])
self._db_wrapper.submit_raids_map_proto(
origin, data["payload"])
self._db_wrapper.submit_spawnpoints_map_proto(
origin, data["payload"])
mon_ids_iv = self.__mitm_mapper.get_mon_ids_iv(origin)
self._db_wrapper.submit_mons_map_proto(origin, data["payload"],
mon_ids_iv)
elif type == 102:
playerlevel = self.__mitm_mapper.playerstats[origin].get_level(
)
if playerlevel >= 30:
logger.info("Processing Encounter received from {} at {}",
str(origin), str(received_timestamp))
self._db_wrapper.submit_mon_iv(origin, received_timestamp,
data["payload"])
else:
logger.debug(
'Playerlevel lower than 30 - not processing encounter Data'
)
elif type == 101:
self._db_wrapper.submit_quest_proto(data["payload"])
elif type == 104:
self._db_wrapper.submit_pokestops_details_map_proto(
data["payload"])
elif type == 4:
self.__mitm_mapper.playerstats[origin].gen_player_stats(
data["payload"])
def _check_pogo_main_screen(self, maxAttempts, again=False):
logger.debug(
"_check_pogo_main_screen: Trying to get to the Mainscreen with {} max attempts...", str(maxAttempts))
pogoTopmost = self._communicator.isPogoTopmost()
if not pogoTopmost:
return False
if not self._takeScreenshot(delayBefore=self.get_devicesettings_value("post_screenshot_delay", 1)):
if again:
logger.error(
"_check_pogo_main_screen: failed getting a screenshot again")
return False
attempts = 0
screenshot_path = self.get_screenshot_path()
if os.path.isdir(screenshot_path):
logger.error(
"_check_pogo_main_screen: screenshot.png/.jpg is not a file/corrupted")
return False
logger.debug("_check_pogo_main_screen: checking mainscreen")
while not self._pogoWindowManager.check_pogo_mainscreen(screenshot_path, self._id):
logger.warning("_check_pogo_main_screen: not on Mainscreen...")
if attempts == maxAttempts:
# could not reach raidtab in given maxAttempts
logger.error(
"_check_pogo_main_screen: Could not get to Mainscreen within {} attempts", str(maxAttempts))
return False
found = self._pogoWindowManager.check_close_except_nearby_button(self.get_screenshot_path(), self._id,
self._communicator, close_raid=True)
if found:
logger.debug("_check_pogo_main_screen: Found (X) button (except nearby)")
if not found and self._pogoWindowManager.look_for_button(screenshot_path, 2.20, 3.01, self._communicator):
logger.debug("_check_pogo_main_screen: Found button (small)")
found = True
if not found and self._pogoWindowManager.look_for_button(screenshot_path, 1.05, 2.20, self._communicator):
logger.debug("_check_pogo_main_screen: Found button (big)")
time.sleep(5)
found = True
logger.debug("_check_pogo_main_screen: Previous checks found popups: {}", str(found))
self._takeScreenshot(delayBefore=self.get_devicesettings_value("post_screenshot_delay", 1))
attempts += 1
logger.debug("_check_pogo_main_screen: done")
return True
def _wait_for_data(self, timestamp, proto_to_wait_for=106, timeout=False):
if not timeout:
timeout = self._devicesettings.get("mitm_wait_timeout", 45)
logger.info('Waiting for data after {}',
datetime.fromtimestamp(timestamp))
data_requested = None
while data_requested is None and timestamp + timeout >= math.floor(
time.time()):
latest = self._mitm_mapper.request_latest(self._id)
data_requested = self._wait_data_worker(latest, proto_to_wait_for,
timestamp)
time.sleep(1)
if data_requested is not None:
logger.info('Got the data requested...')
self._reboot_count = 0
self._restart_count = 0
self._rec_data_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
else:
# TODO: timeout also happens if there is no useful data such as mons nearby in mon_mitm mode, we need to
# TODO: be more precise (timeout vs empty data)
logger.warning("Timeout waiting for data")
current_routemanager = self._walker_routemanager
self._restart_count += 1
restart_thresh = self._devicesettings.get("restart_thresh", 5)
reboot_thresh = self._devicesettings.get("reboot_thresh", 3)
if current_routemanager is not None:
if self._init:
restart_thresh = self._devicesettings.get(
"restart_thresh", 5) * 2
reboot_thresh = self._devicesettings.get(
"reboot_thresh", 3) * 2
if self._restart_count > restart_thresh:
self._reboot_count += 1
if self._reboot_count > reboot_thresh \
and self._devicesettings.get("reboot", False):
logger.error("Rebooting {}", str(self._id))
self._reboot()
raise InternalStopWorkerException
self._restart_count = 0
self._restart_pogo(True)
self.worker_stats()
return data_requested
async def __send_and_wait_internal(self, id, worker_instance, message,
timeout):
async with self.__users_mutex:
user_entry = self.__current_users.get(id, None)
if user_entry is None or user_entry[
1] != worker_instance and worker_instance != 'madmin':
raise WebsocketWorkerRemovedException
message_id = await self.__get_new_message_id()
message_event = asyncio.Event()
message_event.clear()
await self.__set_request(message_id, message_event)
to_be_sent = u"%s;%s" % (str(message_id), message)
logger.debug("To be sent: {}", to_be_sent.strip())
await self.__send(id, to_be_sent)
# now wait for the response!
result = None
logger.debug("Timeout: {}", str(timeout))
event_triggered = None
try:
event_triggered = await asyncio.wait_for(message_event.wait(),
timeout=timeout)
except asyncio.TimeoutError as te:
logger.warning("Timeout, increasing timeout-counter")
# TODO: why is the user removed here?
new_count = await self.__increase_fail_counter(id)
if new_count > 5:
logger.error(
"5 consecutive timeouts to {} or origin is not longer connected, cleanup",
str(id))
await self.__internal_clean_up_user(id, None)
await self.__reset_fail_counter(id)
await self.__remove_request(message_id)
raise WebsocketWorkerTimeoutException
if event_triggered:
logger.debug("Received answer in time, popping response")
await self.__reset_fail_counter(id)
result = await self.__pop_response(message_id)
if isinstance(result, str):
logger.debug("Response to {}: {}", str(id),
str(result.strip()))
else:
logger.debug("Received binary data to {}, starting with {}",
str(id), str(result[:10]))
return result
def activate_existing_neo4j_db(database_name):
"""
activates a neo4j database that has been previously imported
@param {string} database_name which has the .db extension included
@return {bool} whether or not the given database name is activated
"""
# stop neo4j
STOP_NEO4J_COMMAND = "neo4j stop"
run_os_command(STOP_NEO4J_COMMAND)
db_absolute_path = os.path.join(constantsModule.NEO4J_DB_PATH,
database_name)
if not os.path.exists(db_absolute_path):
if constantsModule.DEBUG_PRINTS:
logger.warning(
"No neo4j database with name \'%s\' exists for activation!" %
database_name)
return False
else:
# change the active db
new_config_line = "dbms.active_database=%s" % database_name
if constantsModule.CURRENT_PLATFORM == constantsModule.PLATFORMS[
'MAC_OS_X']:
CHANGE_ACTIVE_GRAPH_COMMAND = """sed -i '' 's/dbms.active_database=.*db/%s/1' %s""" % (
new_config_line, constantsModule.NEO4J_CONF)
elif constantsModule.CURRENT_PLATFORM == constantsModule.PLATFORMS[
'Linux']:
CHANGE_ACTIVE_GRAPH_COMMAND = """sed -i 's/dbms.active_database=.*db/%s/1' %s""" % (
new_config_line, constantsModule.NEO4J_CONF)
else:
logger.error(
'Detected unsupported platform. Check your enviornment variables (.env) file if your platform is supported but it is set wrong.'
)
return False
run_os_command(CHANGE_ACTIVE_GRAPH_COMMAND, print_stdout=False)
# start neo4j
START_NEO4J_COMMAND = "neo4j start"
run_os_command(START_NEO4J_COMMAND)
if constantsModule.DEBUG_PRINTS:
logger.info("Neo4J DB setup successful.")
time.sleep(3)
return True
def update_latest(self, origin, timestamp, key, values_dict):
updated = False
self.__mapping_mutex.acquire()
if origin in self.__mapping.keys():
logger.debug("Updating timestamp of {} with method {} to {}", str(
origin), str(key), str(timestamp))
self.__mapping[origin][key] = {}
self.__mapping[origin][key]["timestamp"] = timestamp
self.__mapping[origin][key]["values"] = values_dict
updated = True
else:
logger.warning(
"Not updating timestamp of {} since origin is unknown", str(origin))
self.__mapping_mutex.release()
return updated
def get_monlist(self, listname, areaname):
if type(listname) is list:
logger.error(
'Area {} is using old list format instead of global mon list. Please check your mappings.json.'
' Using empty list instead.'.format(str(areaname)))
return []
if listname is not None and listname in self._monlists:
return self._monlists[listname]
elif listname is None:
return []
else:
logger.warning(
"IV list '{}' has been used in area '{}' but does not exist. Using empty IV list instead.",
listname, areaname)
return []
def switchtimer(self):
logger.info('[{}] - Starting Switchtimer', str(self._id))
switchtime = self._switchtime
sts1 = switchtime[0].split(':')
sts2 = switchtime[1].split(':')
while not self.__stop_switchtimer.is_set():
tmFrom = datetime.datetime.now().replace(hour=int(sts1[0]),
minute=int(sts1[1]),
second=0,
microsecond=0)
tmTil = datetime.datetime.now().replace(hour=int(sts2[0]),
minute=int(sts2[1]),
second=0,
microsecond=0)
tmNow = datetime.datetime.now()
# check if current time is past start time
# and the day has changed already. thus shift
# start time back to the day before
if tmFrom > tmTil > tmNow:
tmFrom = tmFrom + datetime.timedelta(days=-1)
# check if start time is past end time thus
# shift start time one day into the future
if tmTil < tmFrom:
tmTil = tmTil + datetime.timedelta(days=1)
if tmFrom <= tmNow < tmTil:
logger.info('[{}] - Switching Mode', str(self._id))
self.set_switch(True)
while self.get_switch():
tmNow = datetime.datetime.now()
logger.info("[{}] - Currently in switchmode",
str(self._id))
if tmNow >= tmTil:
logger.warning(
'[{}] - Switching back - here we go ...',
str(self._id))
self.set_switch(False)
if self.__stop_switchtimer.is_set():
logger.info("[{}] switchtimer stopping in switchmode",
str(self._id))
self.set_switch(False)
time.sleep(30)
time.sleep(30)
logger.info("[{}] switchtimer stopping", str(self._id))
def get_to_be_encountered(self, geofence_helper, min_time_left_seconds,
eligible_mon_ids: Optional[List[int]]):
if min_time_left_seconds is None or eligible_mon_ids is None:
logger.warning(
"DbWrapper::get_to_be_encountered: Not returning any encounters since no time left or "
"eligible mon IDs specified")
return []
logger.debug("Getting mons to be encountered")
query = (
"SELECT latitude, longitude, encounter_id, spawnpoint_id, pokemon_id, "
"TIMESTAMPDIFF(SECOND, UTC_TIMESTAMP(), disappear_time) AS expire "
"FROM pokemon "
"WHERE individual_attack IS NULL AND individual_defense IS NULL AND individual_stamina IS NULL "
"AND encounter_id != 0 "
"and (disappear_time BETWEEN DATE_ADD(UTC_TIMESTAMP(), INTERVAL %s SECOND) "
"and DATE_ADD(UTC_TIMESTAMP(), INTERVAL 60 MINUTE))"
"ORDER BY expire ASC")
vals = (int(min_time_left_seconds), )
results = self.execute(query, vals, commit=False)
next_to_encounter = []
for latitude, longitude, encounter_id, spawnpoint_id, pokemon_id, expire in results:
if pokemon_id not in eligible_mon_ids:
continue
elif latitude is None or longitude is None:
logger.warning("lat or lng is none")
continue
elif geofence_helper and not geofence_helper.is_coord_inside_include_geofence(
[latitude, longitude]):
logger.debug(
"Excluded encounter at {}, {} since the coordinate is not inside the given include fences",
str(latitude), str(longitude))
continue
next_to_encounter.append(
(pokemon_id, Location(latitude, longitude), encounter_id))
# now filter by the order of eligible_mon_ids
to_be_encountered = []
i = 0
for mon_prio in eligible_mon_ids:
for mon in next_to_encounter:
if mon_prio == mon[0]:
to_be_encountered.append((i, mon[1], mon[2]))
i += 1
return to_be_encountered
def post(self, request, *args, **kwargs):
data = request.data.dict() # 回调参数，是QueryDict类型，不能直接调用pop方法
sign = data.pop('sign') # 签名
out_trade_no = data.get('out_trade_no') # 订单号
result = alipay.verify(data, sign)
if result and data["trade_status"] in ("TRADE_SUCCESS", "TRADE_FINISHED"):
try:
order = models.Order.objects.get(out_trade_no=out_trade_no)
if order.order_status != 1:
order.order_status = 1
order.save()
logger.warning('%s订单完成支付' % out_trade_no)
return Response('success')
except:
pass
return Response('failed')
def _reboot(self):
if not self.get_devicesettings_value("reboot", True):
logger.warning(
"Reboot command to be issued to device but reboot is disabled. Skipping reboot"
)
return True
try:
start_result = self._communicator.reboot()
except WebsocketWorkerRemovedException:
logger.error(
"Could not reboot due to client already having disconnected")
start_result = False
time.sleep(5)
self._db_wrapper.save_last_reboot(self._id)
self.stop_worker()
return start_result
def received_data_worker(self):
while True:
item = self._data_queue.get()
items_left = self._data_queue.qsize()
logger.debug(
"MITM data processing worker retrieved data. Queue length left afterwards: {}",
str(items_left))
if items_left > 50: # TODO: no magic number
logger.warning(
"MITM data processing workers are falling behind! Queue length: {}",
str(items_left))
if item is None:
logger.warning("Received none from queue of data")
break
self.process_data(item[0], item[1], item[2])
self._data_queue.task_done()
def process_step_collect_parse(project,
step,
process_result,
format_args=None):
"""
Function will parse the file from an output
:type step: structures.project_step.ProjectStep
:type project: structures.project.Project
:type process_result: proc.step.step_shell.ProcessStepResult
"""
logger.debug('parsing output artifacts')
if not process_result.output:
logger.warning(
'Empty output received, make sure the field ouutput is set to \n'
'output: log+stdout \n'
'in order to capture output of the shell step')
return []
index = 0
output = process_result.output
start = step.collect.parse.start
stop = step.collect.parse.start
ls, le = len(start), len(stop)
length = len(output)
reports = list()
while True:
# no more reports found
s = output.find(start, index, length)
if s == -1:
break
# broken report
e = output.find(stop, s + 1, length)
if e == -1:
logger.debug(
'could not find end of the report file while parsing the output'
)
break
index = e + le + 1
# try to parse the data or skip it
reports.append(output[s + ls:e].strip())
return reports
def __call__(self, *args):
logger.debug3("HTTP Request from {}".format(str(request.remote_addr)))
origin = request.headers.get('Origin')
abort = False
if request.url_rule is not None and str(
request.url_rule) == '/status/':
auth = request.headers.get('Authorization', False)
if self.application_args.mitm_status_password != "" and \
(not auth or auth != self.application_args.mitm_status_password):
self.response = Response(status=500, headers={})
abort = True
else:
abort = False
else:
if not origin:
logger.warning("Missing Origin header in request")
self.response = Response(status=500, headers={})
abort = True
elif (
self.mapping_manager.get_all_devicemappings().keys()
is not None and
(origin is None or origin
not in self.mapping_manager.get_all_devicemappings().keys())):
logger.warning(
"MITMReceiver request without Origin or disallowed Origin: {}"
.format(origin))
self.response = Response(status=403, headers={})
abort = True
elif self.mapping_manager.get_auths() is not None:
auth = request.headers.get('Authorization', None)
if auth is None or not check_auth(
auth, self.application_args,
self.mapping_manager.get_auths()):
logger.warning("Unauthorized attempt to POST from {}",
str(request.remote_addr))
self.response = Response(status=403, headers={})
abort = True
if not abort:
try:
# TODO: use response data
if len(request.data) > 0:
request_data = json.loads(request.data)
else:
request_data = {}
response_payload = self.action(origin, request_data)
if response_payload is None:
response_payload = ""
self.response = Response(
status=200, headers={"Content-Type": "application/json"})
self.response.data = response_payload
except Exception as e: # TODO: catch exact exception
logger.warning("Could not get JSON data from request: {}",
str(e))
self.response = Response(status=500, headers={})
return self.response
def _reboot(self, mitm_mapper: Optional[MitmMapper]=None):
if not self._devicesettings.get("reboot", True):
logger.warning("Reboot command to be issued to device but reboot is disabled. Skipping reboot")
return True
try:
start_result = self._communicator.reboot()
except WebsocketWorkerRemovedException:
logger.error(
"Could not reboot due to client already having disconnected")
start_result = False
time.sleep(5)
if mitm_mapper is not None:
mitm_mapper.collect_location_stats(self._id, self.current_location, 1, time.time(), 3, 0,
self._walker_routemanager.get_walker_type(), 99)
self._db_wrapper.save_last_reboot(self._id)
self.stop_worker()
return start_result
def update_latest(self,
origin: str,
key: str,
values_dict,
timestamp_received_raw: float = None,
timestamp_received_receiver: float = None):
if timestamp_received_raw is None:
timestamp_received_raw = time.time()
if timestamp_received_receiver is None:
timestamp_received_receiver = time.time()
updated = False
logger.debug3(
"Trying to acquire lock and update proto {} received by {}".format(
origin, key))
with self.__mapping_mutex:
if origin not in self.__mapping.keys(
) and origin in self.__mapping_manager.get_all_devicemappings(
).keys():
logger.info(
"New device detected, {}. Setting up the device configuration",
origin)
self.__add_new_device(origin)
if origin in self.__mapping.keys():
logger.debug("Updating timestamp of {} with method {} to {}",
str(origin), str(key),
str(timestamp_received_raw))
if self.__mapping.get(origin) is not None and self.__mapping[
origin].get(key) is not None:
del self.__mapping[origin][key]
self.__mapping[origin][key] = {}
self.__mapping[origin][key][
"timestamp"] = timestamp_received_raw
self.__mapping[origin][
"timestamp_last_data"] = timestamp_received_raw
self.__mapping[origin][
"timestamp_receiver"] = timestamp_received_receiver
self.__mapping[origin][key]["values"] = values_dict
updated = True
else:
logger.warning(
"Not updating timestamp of {} since origin is unknown",
str(origin))
logger.debug3("Done updating proto {} of {}".format(key, origin))
return updated
def send_and_wait(self, id, worker_instance, message, timeout):
logger.debug("{} sending command: {}", str(id), message.strip())
self.__current_users_mutex.acquire()
user_entry = self.__current_users.get(id, None)
self.__current_users_mutex.release()
if user_entry is None or user_entry[
1] != worker_instance and worker_instance != 'madmin':
raise WebsocketWorkerRemovedException
message_id = self.__get_new_message_id()
message_event = Event()
message_event.clear()
self.__set_request(message_id, message_event)
to_be_sent = u"%s;%s" % (str(message_id), message)
logger.debug("To be sent: {}", to_be_sent.strip())
self.__send(id, to_be_sent)
# now wait for the response!
result = None
logger.debug("Timeout: {}", str(timeout))
if message_event.wait(timeout):
logger.debug("Received answer in time, popping response")
self.__reset_fail_counter(id)
result = self.__pop_response(message_id)
if isinstance(result, str):
logger.debug("Response to {}: {}", str(id),
str(result.strip()))
else:
logger.debug("Received binary data to {}, starting with {}",
str(id), str(result[:10]))
else:
# timeout reached
logger.warning("Timeout, increasing timeout-counter")
# TODO: why is the user removed here?
new_count = self.__increase_fail_counter(id)
if new_count > 5:
logger.error("5 consecutive timeouts to {}, cleanup", str(id))
# TODO: signal worker to stop and NOT cleanup the websocket by itself!
self.clean_up_user(id, None)
raise WebsocketWorkerTimeoutException
self.__remove_request(message_id)
return result
def _reboot(self, mitm_mapper: Optional[MitmMapper] = None):
if not self.get_devicesettings_value("reboot", True):
logger.warning("Skipping reboot, disabled in device settings.")
return True
try:
start_result = self._communicator.reboot()
except WebsocketWorkerRemovedException:
logger.error(
"Could not reboot due to client already disconnected")
start_result = False
time.sleep(5)
if mitm_mapper is not None:
mitm_mapper.collect_location_stats(self._id, self.current_location, 1, time.time(), 3, 0,
self._mapping_manager.routemanager_get_mode(self._routemanager_name),
99)
self._db_wrapper.save_last_reboot(self._id)
self.stop_worker()
return start_result
def send_sms(mobile, code, exp):
try:
# 发送短信
response = sender.send_with_param(MOBILE_PREFIX,
mobile,
TEMPLATE_ID, (code, exp),
sign=SMS_SING,
extend="",
ext="")
# 成功
if response and response['result'] == 0:
return True
# 失败
logger.warning('%s - %s' % ('短信发送失败', response['result']))
except Exception as e:
# 异常
logger.warning('%s - %s' % ('短信发送失败', e))
return False
def proto_endpoint(self, origin, data):
type = data.get("type", None)
if type is None or type == 0:
logger.warning(
"Could not read method ID. Stopping processing of proto")
return None
if not self.__mitm_mapper.get_injection_status(origin):
logger.info("Worker {} is injected now", str(origin))
self.__mitm_mapper.set_injection_status(origin)
# extract timestamp from data
timestamp: float = data.get("timestamp", int(math.floor(time.time())))
self.__mitm_mapper.update_latest(
origin,
timestamp_received_raw=timestamp,
timestamp_received_receiver=time.time(),
key=type,
values_dict=data)
self._data_queue.put((timestamp, data, origin))
return None
def _speed_weather_check_thread(self):
while not self._stop_worker_event.is_set():
while not self.__start_speed_weather_check_event.is_set():
time.sleep(0.5)
if self._stop_worker_event.is_set():
return
logger.debug("checkSpeedWeatherWarningThread: acquiring lock")
logger.debug("Speedweather: acquiring lock")
self._work_mutex.acquire()
try:
logger.debug("Speedweather: acquired lock")
logger.debug("checkSpeedWeatherWarningThread: lock acquired")
logger.debug(
"checkSpeedWeatherWarningThread: Checking if pogo is running..."
)
if not self._communicator.isPogoTopmost():
logger.warning(
"checkSpeedWeatherWarningThread: Starting Pogo")
self._restart_pogo()
reached_raidscreen = self._getToRaidscreen(10, True)
if reached_raidscreen:
logger.debug(
"checkSpeedWeatherWarningThread: checkSpeedWeatherWarningThread: reached raidscreen..."
)
self.__start_speed_weather_check_event.clear()
else:
logger.debug(
"checkSpeedWeatherWarningThread: did not reach raidscreen in 10 attempts"
)
self.__start_speed_weather_check_event.set()
except WebsocketWorkerRemovedException as e:
logger.error("Timeout during init of worker {} with {}",
str(self._id), str(e))
self._stop_worker_event.set()
self._work_mutex.release()
return
logger.debug("checkSpeedWeatherWarningThread: releasing lock")
self._work_mutex.release()
logger.debug("Speedweather: released lock")
time.sleep(1)
def _check_routepools(self, timeout: int = 300):
while not self._stop_update_thread.is_set():
routepool_changed: bool = False
logger.debug("Checking routepool for idle/dead workers")
with self._manager_mutex:
for origin in list(self._routepool):
entry: RoutePoolEntry = self._routepool[origin]
if time.time() - entry.last_access > timeout + entry.worker_sleeping:
logger.warning(
"Worker {} has not accessed a location in {} seconds, removing from routemanager",
origin, timeout)
self.unregister_worker(origin)
i = 0
while i < 60 and not self._stop_update_thread.is_set():
if self._stop_update_thread.is_set():
logger.info("Stop checking routepools")
i += 1
time.sleep(1)
def __init__(self, args):
self._args = args
self._useadb = args.use_adb
self._client = None
if self._useadb:
try:
from adb.client import Client as AdbClient
except ImportError:
pass
self.check_adblib = 'adb.client' in sys.modules
if not self.check_adblib:
logger.warning(
"Could not find pure-python-adb library - no support for ADB."
"If you have no idea what ADB is or not planning to use it - IGNORE :)"
)
self._useadb = False
else:
self._client = AdbClient(
host=self._args.adb_server_ip, port=self._args.adb_server_port)
def __check_close_present(self, filename, identifier, communicator, radiusratio=12, Xcord=True):
if not os.path.isfile(filename):
logger.warning(
"__check_close_present: {} does not exist", str(filename))
return False
try:
image = cv2.imread(filename)
height, width, _ = image.shape
except:
logger.error("Screenshot corrupted :(")
return False
cv2.imwrite(os.path.join(self.temp_dir_path,
str(identifier) + '_exitcircle.jpg'), image)
if self.__read_circle_count(os.path.join(self.temp_dir_path, str(identifier) + '_exitcircle.jpg'), identifier,
float(radiusratio), communicator, xcord=False, crop=True, click=True, canny=True) > 0:
return True
def validate_model_opts(cls, model_opt):
assert model_opt.model_type in ["text", "img", "audio"], \
"Unsupported model type %s" % model_opt.model_type
# this check is here because audio allows the encoder and decoder to
# be different sizes, but other model types do not yet
same_size = model_opt.enc_rnn_size == model_opt.dec_rnn_size
assert model_opt.model_type == 'audio' or same_size, \
"The encoder and decoder rnns must be the same size for now"
assert model_opt.rnn_type != "SRU" or model_opt.gpu_ranks, \
"Using SRU requires -gpu_ranks set."
if model_opt.share_embeddings:
if model_opt.model_type != "text":
raise AssertionError(
"--share_embeddings requires --model_type text.")
if model_opt.model_dtype == "fp16":
logger.warning(
"FP16 is experimental, the generated checkpoints may "
"be incompatible with a future version")
def check_auth(authHeader, args, auths):
if "Basic" not in authHeader:
logger.warning("Auth without Basic auth, aborting.")
return False
try:
base64raw = authHeader.replace("Basic", "").replace(" ", "")
decoded = str(base64.b64decode(base64raw))
except TypeError:
return False
decodedSplit = decoded.split(":")
# decoded.split(":")
if args is not None and auths is not None:
# check if user is present in ws_auth, if so also check pw
# god awful dirty replace... no idea why the b' is in there...
username = str(decodedSplit[0]).replace("b'", "")
passwordInConf = auths.get(username, None)
if passwordInConf is None or passwordInConf is not None and passwordInConf != decodedSplit[1].replace("'", ""):
logger.warning("Auth attempt from {} failed", str(authHeader))
return False
return True
def main():
parser = argparse.ArgumentParser(
description=
'This script analyzes a given website collected by the JAW crawler to detect client-side CSRF.'
)
parser.add_argument(
'--siteid',
help=
'Identifier of the site to be tested, located under /hpg_construction/outputs/<WEBSITE>',
default='')
# dictionary of the provided arguments
args = vars(parser.parse_args())
website_id = args['siteid']
if website_id != '' and webpage_url != '':
driver_program_web_site(website_id)
else:
logger.warning('input parameter siteid is not valid.')
def __prepare_quest_data(self, quest_data):
ret = []
for stopid in quest_data:
stop = quest_data[str(stopid)]
if self.__is_in_excluded_area(
[stop["latitude"], stop["longitude"]]):
continue
try:
quest = generate_quest(quest_data[str(stopid)])
quest_payload = self.__construct_quest_payload(quest)
entire_payload = {"type": "quest", "message": quest_payload}
ret.append(entire_payload)
except Exception as e:
logger.warning(
"Exception occured while generating quest webhook: {}",
str(e))
return ret
def worker_stats(self):
logger.debug('===============================')
logger.debug('Worker Stats')
logger.debug('Origin: {}', str(self._id))
logger.debug('Routemanager: {}', str(self._routemanager_name))
logger.debug('Restart Counter: {}', str(self._restart_count))
logger.debug('Reboot Counter: {}', str(self._reboot_count))
logger.debug('Reboot Option: {}', str(
self.get_devicesettings_value("reboot", False)))
logger.debug('Current Pos: {} {}', str(
self.current_location.lat), str(self.current_location.lng))
logger.debug('Last Pos: {} {}', str(
self.last_location.lat), str(self.last_location.lng))
routemanager_status = self._mapping_manager.routemanager_get_route_stats(self._routemanager_name, self._id)
if routemanager_status is None:
logger.warning("Routemanager not available")
routemanager_status = [None, None]
else:
logger.debug('Route Pos: {} - Route Length: {}', str(routemanager_status[0]), str(routemanager_status[1]))
routemanager_init: bool = self._mapping_manager.routemanager_get_init(self._routemanager_name)
logger.debug('Init Mode: {}', str(routemanager_init))
logger.debug('Last Date/Time of Data: {}', str(self._rec_data_time))
logger.debug('===============================')
dataToSave = {
'Origin': self._id,
'Routemanager': str(self._routemanager_name),
'RebootCounter': str(self._reboot_count),
'RestartCounter': str(self._restart_count),
'RebootingOption': str(self.get_devicesettings_value("reboot", False)),
'CurrentPos': str(self.current_location.lat) + ", " + str(self.current_location.lng),
'LastPos': str(self.last_location.lat) + ", " + str(self.last_location.lng),
'RoutePos': str(routemanager_status[0]),
'RouteMax': str(routemanager_status[1]),
'Init': str(routemanager_init),
'LastProtoDateTime': str(self._rec_data_time),
'CurrentSleepTime': str(self._current_sleep_time)
}
self._db_wrapper.save_status(dataToSave)
