def get_test_validators(test_name, tests_dir): try: test_subtasks = tu.get_test_subtasks_from_tests_dir( test_name, tests_dir) except tu.MalformedTestsException as e: sys.stderr.write("{}\n".format(e)) sys.exit(3) if len(test_subtasks) == 0: log_warning("Test '%s' is in no subtasks." % test_name) data = load_json(SUBTASKS_JSON) global_validators = data.get('global_validators', []) subtask_sensitive_validators = data.get('subtask_sensitive_validators', []) def check_subtask_sensitive_validators(subtask_sensitive_validators): subtask_placeholder_var = "subtask" subtask_placeholder_test_substitute = "___SUBTASK_PLACEHOLDER_SUBSTITUTE___" for subtask_sensitive_validator in subtask_sensitive_validators: try: subtask_validator_substituted = subtask_sensitive_validator.format( **{ subtask_placeholder_var: subtask_placeholder_test_substitute }) except KeyError as e: sys.stderr.write( 'Subtask-sensitive validator "{}" contains unknown placeholder {{{}}}.\n' .format(subtask_sensitive_validator, e.args[0])) sys.exit(3) else: if subtask_placeholder_test_substitute not in subtask_validator_substituted: log_warning( 'Subtask-sensitive validator "{}" does not contain the subtask placeholder {{{}}}.' .format(subtask_sensitive_validator, subtask_placeholder_var)) check_subtask_sensitive_validators(subtask_sensitive_validators) test_validators = list(global_validators) for subtask in test_subtasks: test_validators += [ validator.format(subtask=subtask) for validator in subtask_sensitive_validators ] test_validators += navigate_json(data, 'subtasks/%s' % subtask, SUBTASKS_JSON).get('validators', []) if len(test_validators) == 0: log_warning("There is no validator for test {}.".format(test_name)) def unify_list(l): seen = [] for e in l: if e not in seen: seen.append(e) return seen return unify_list(test_validators)
def wait_for_disarm(vehicle, timeout=180): """ Wait for the vehicle to disarm and log every 2 sec. Args: vehicle(dronekit.Vehicle): the copter to be controlled. timeout(int): seconds before invoking forced exit. Default 30 seconds. """ wait_count = 0 sleep_period = 1 log_period = 30 timeout_limit = timeout / sleep_period log_count = log_period / sleep_period while vehicle.armed: if wait_count % log_count == 0: # 10sec period logging util.log_info("Waiting for the copter to disarm.") if wait_count >= timeout_limit: util.log_warning("Wait timeout! Exit script now!") break time.sleep(sleep_period) wait_count = wait_count + 1
def set_mode(vehicle, mode): """ Set the vehicle's flight modes. 200ms period state validation. Args: vehicle(dronekit.Vehicle): the vehicle to be controlled. mode(str): flight mode string, supported by the firmware. Returns: bool: True if success, False if failed. Failure will set shared.status['abort']. """ util.log_info("Setting %s." % mode) shared.status['manual_mode'] = mode vehicle.mode = VehicleMode(mode) wait_count = 0 while True: time.sleep(.2) wait_count = wait_count + 1 if vehicle.mode.name == mode: return True elif wait_count >= 45: util.log_warning("Unable to set %s. Assume link lost." % mode) shared.status['abort'] = True return False elif wait_count % 15 == 0: util.log_warning("Retry setting %s" % mode) vehicle.mode = VehicleMode(mode) # resend command
def convert_origin_func_to_npu(node, origin_func, org_func_name, params_list, is_class_func=None): """Convert original Tensorflow function to NPU function""" if not check_func_arguments(origin_func, node.args, node.keywords, is_class_func): return node if org_func_name == "Estimator.train": content = "".join([ util_global.get_value('path'), ":", str(getattr(node, "lineno", "None")) ]) while True: message = input( "Check if the train function in " + content + " is the Estimator train function. If yes, " "enter 'y' to perform distributed porting on the train function. if no, enter 'n': " ) if message == "y": break if message == "n": log_warning("".join([ "The train func in ", content, " is user-defined functions, will not perform distributed porting" ])) return node print("Input is error, Please enter 'y' or 'n'.") for param_name in params_list: node = match_func_params_and_convert(node, origin_func, org_func_name, param_name, is_class_func) util_global.set_value('need_conver', True) return node
def conver(): """The entry point to convert Tensorflow script""" print("Begin conver, input file: " + util_global.get_value('input') + '\n') out_path = util_global.get_value('output') dst_path = os.path.split(util_global.get_value('input').rstrip('\\/'))[-1] dst_path_new = dst_path + util_global.get_value('timestap') conver_path = os.walk(util_global.get_value('input')) report_dir = util_global.get_value('report') mkdir(report_dir) report_xlsx = os.path.join(report_dir, 'api_analysis_report.xlsx') util_global.set_value('generate_dir_report', pd.DataFrame()) for path, _, file_list in conver_path: for file_name in file_list: out_path_dst = abs_join( dst_path_new, path.split(util_global.get_value('input'))[1]) file_path = os.path.join(path, file_name).replace('\\', '/') if not check_path_length(file_path): content = "".join([ "The file:", file_path, " length is invalid, skip convert." ]) log_warning(content) continue content = "".join(["Begin conver file: ", file_path]) print(content) threshold_file_size = 10 * 1024 * 1024 if file_name.endswith(".py"): if os.path.getsize(file_path) > threshold_file_size: content = "".join([ "The file:", file_path, " size is over 10M, skip convert." ]) log_warning(content) continue util_global.set_value('path', file_path) mkdir(os.path.join(out_path, out_path_dst)) conver_ast(path, out_path_dst, file_name) if util_global.get_value('need_conver', False): content = "".join( ["Finish conver file: ", file_path, '\n']) print(content) write_report_terminator(content) else: mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) else: mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) adjust_index() analysis_report = util_global.get_value('generate_dir_report') if analysis_report.empty: print('No api data in the report') else: analysis_report.to_excel(report_xlsx, index=True) get_api_statistic(analysis_report) print("Finish conver, output file: " + out_path + "; report file: " + util_global.get_value('report'))
def read_ignore_list(path): apps = [] if not os.path.exists(path): log_warning("Can't not find the ignore list at '%s'" % path) return apps with io.open(path, 'r', encoding='utf8') as f: for line in f: app_id = line.split(' ')[0] apps.append(int(app_id)) return apps
def convert_tf_gradient_distributed(node): """Convert Tensorflow gradient APIs in distributed mode""" content = "".join([ util_global.get_value('path'), ":", str(getattr(node, "lineno", "None")), " is tf.gradient api, tool inserts npu_allreduce after computing grads by default.", " You can adjust the allreduce position according to the algorithm" ]) log_warning(content) new_node = ast.Call(func=ast.Name(id="npu_allreduce", ctx=ast.Load()), args=[node], keywords=[]) ast.copy_location(new_node, node) util_global.set_value("need_conver", True) return new_node
def check_subtask_sensitive_validators(subtask_sensitive_validators): subtask_placeholder_var = "subtask" subtask_placeholder_test_substitute = "___SUBTASK_PLACEHOLDER_SUBSTITUTE___" for subtask_sensitive_validator in subtask_sensitive_validators: try: subtask_validator_substituted = subtask_sensitive_validator.format( **{ subtask_placeholder_var: subtask_placeholder_test_substitute }) except KeyError as e: sys.stderr.write( 'Subtask-sensitive validator "{}" contains unknown placeholder {{{}}}.\n' .format(subtask_sensitive_validator, e.args[0])) exit(3) else: if subtask_placeholder_test_substitute not in subtask_validator_substituted: log_warning( 'Subtask-sensitive validator "{}" does not contain the subtask placeholder {{{}}}.' .format(subtask_sensitive_validator, subtask_placeholder_var))
def __init__(self, id): self.REQUESTING = True self.id = id self.generated = 0 while self.REQUESTING: try: self.html = requests.get("http://www.puzzledragonx.com/en/monster.asp?n=%s" % self.id).content self.parsed_html = BeautifulSoup(self.html, from_encoding="utf-8") self.name = self.parsed_html.body.find('h1').text self.get_data() self.generated = int(strftime("%Y%m%d", gmtime())) self.status = 1 self.REQUESTING = False self.html = None self.parsed_html = None if self.name == "Puzzle Dragon X": self.status = 0 except Exception as e: log_warning(e) self.status = 0 sleep(SLEEP_LONG)
def _check_terminate_condition(vehicle, name_string): """ Check several termination conditions and set the stopflag to a thread. Priority: shared.status['abort'] == True: emergency, highest level. shared.status['thread_flag'] has FLOCKING_FLAG: thread killing command. vehicle.mode != 'GUIDED': grabbed manual control. Args: vehicle(dronekit.Vehicle): the copter to be controlled. name_string(str): a string specifies the thread name. Returns: bool: True if condition is met, False if no-stop. """ status = False exit_number = 0 if shared.status['abort']: util.log_warning("Abort detected!") status = True exit_number = 1 elif shared.status['thread_flag'] & shared.FLOCKING_FLAG: util.log_info("FLOCKING_FLAG detected.") status = True exit_number = 2 elif vehicle.mode != 'GUIDED': util.log_warning("Mode switched out from 'GUIDED'!") status = True exit_number = 3 if status: util.log_info("Stopping %s" % name_string) return [status, exit_number]
def arm_and_takeoff(vehicle, target_alt, loiter_time): """ Arm the vehicle and takeoff to target altitude. This is a blocking check, so the program is stuck indefinately unless received `LIFT` clearance, stopped by `EXIT` command or killed by KeyboardInterrupt. Args: vehicle(dronekit.Vehicle): the copter to be controlled. target_alt(float): takeoff altitude, validated in the main script. loiter_time(int): seconds to wait after `des_alt` is reached. Returns: bool: True if success, False if killed by `EXIT` command. """ util.log_info("Checking if armable.") wait_count = 0 while not vehicle.is_armable: time.sleep(.2) wait_count = wait_count + 1 if wait_count % 25 == 0: util.log_warning("Vehicle not armable.") if wait_count >= 100: util.log_warning("Unable to arm. Abort.") shared.status['abort'] = True return False util.log_info("Switching to GUIDED and Arming.") set_mode(vehicle, "GUIDED") util.log_debug("Arming...") vehicle.armed = True time.sleep(3) wait_count = 0 while True: time.sleep(.5) wait_count = wait_count + 1 if vehicle.armed: util.log_info("Armed.") break elif wait_count % 10 == 0: util.log_warning('Retry arming.') if wait_count >= 20: util.log_warning("Arming failed. Abort.") shared.status['abort'] = True return False vehicle.simple_takeoff(target_alt) # Wait until the vehicle reaches a safe altitude (95%), or otherwise the command # after Vehicle.simple_takeoff will execute immediately, causing unexpected results. while True: util.log_debug("Altitude: %s" % vehicle.location.global_relative_frame.alt) if vehicle.location.global_relative_frame.alt >= target_alt - shared.ALT_TOLERANCE: util.log_info("Target altitude reached: %s m." % vehicle.location.global_relative_frame.alt) break time.sleep(.5) time.sleep(loiter_time) return True
def _preflight_check(vehicle, xbee): """ Perform preflight check, validate home coordinates and satellites fix. This is a blocking check, so the program is stuck indefinately unless stopped by `EXIT` command or killed by KeyboardInterrupt. Args: vehicle(dronekit.Vehicle): the vehicle to be controlled. xbee(xbee.Zigbee): the XBee communication interface. Returns: bool: True if success, False if killed by `EXIT` command. """ util.log_info("Waiting for home location.") while not vehicle.home_location: if shared.status['command'] == 'EXIT': comm.xbee_broadcast(xbee, "IFO,%s abort takeoff." % shared.AGENT_ID) util.log_info("'EXIT' received. Abort takeoff.") return False time.sleep(2) cmds = vehicle.commands cmds.download() cmds.wait_ready() comm.xbee_broadcast(xbee, "IFO,%s getting home fix." % shared.AGENT_ID) # We have a home location now. comm.xbee_broadcast( xbee, 'IFO,%s home: %s.' % (shared.AGENT_ID, vehicle.home_location)) util.log_info('Home location: %s' % vehicle.home_location) # Check satellite condition to ensure 3D-fix first while vehicle.gps_0.fix_type < 3: if shared.status['command'] == 'EXIT': comm.xbee_broadcast(xbee, "IFO,%s abort takeoff." % shared.AGENT_ID) util.log_info("'EXIT' received. Abort takeoff.") return False comm.xbee_broadcast(xbee, "IFO,%s GNSS No 3D-fix." % shared.AGENT_ID) util.log_warning("GNSS No 3D Fix.") time.sleep(3) # APM:Copter parameter: GPS_HDOP_GOOD # The value is mutiplied by 100 into a integer, default good HDOP is below 140 while vehicle.gps_0.eph > 140 or vehicle.gps_0.satellites_visible < 9: if shared.status['command'] == 'EXIT': comm.xbee_broadcast(xbee, "IFO,%s abort takeoff." % shared.AGENT_ID) util.log_info("'EXIT' received. Abort takeoff.") return False util.log_info( "HDOP: %.2f NumSat: %s" % (vehicle.gps_0.eph / 100.0, vehicle.gps_0.satellites_visible)) comm.xbee_broadcast( xbee, "IFO,%s HDOP: %.2f NumSat: %s" % (shared.AGENT_ID, vehicle.gps_0.eph / 100.0, vehicle.gps_0.satellites_visible)) time.sleep(3) # --END of while-- Preflight check passed. comm.xbee_broadcast(xbee, "IFO,%s Preflight check passed." % shared.AGENT_ID) util.log_info( "Preflight check passed. HDOP: %.2f NumSats: %s" % (vehicle.gps_0.eph / 100.0, vehicle.gps_0.satellites_visible)) util.log_info("Local time %s" % shared.timestamp) return True
for (app_id, app_name) in apps: count += 1 file_path = meta_dir_path + "/app_meta_" + str(app_id) + ".json" # Check if the metadata exists if (not args.re_meta) and os.path.exists(file_path): continue # Downloads the json data data = retrieve_data(APP_URL_PREFIX + str(app_id) + "&cc=us") if data is None: continue if data[str(app_id)]["success"] is False: log_warning("Failed to retrieve the data for '%s' (id: %d)" % (app_name, app_id)) continue data = data[str(app_id)]["data"] # Only keep the attributes we care about new_data = {} for attr in APP_ATTRS: if attr in data: new_data[attr] = data[attr] # Save to the file with open(file_path, "w") as f: f.write(json.dumps(new_data)) down_count += 1 if down_count % 10 == 0:
help="print the logs") args = parser.parse_args() # Connect with IPC fed by exchange_md.py ctx = zmq.Context() pullEndpoint = "ipc:///home/ubuntu/tmp/.md_server_private_zmq_address" pullSocket = ctx.socket(zmq.PULL) pullSocket.bind(pullEndpoint) pullSocket.setsockopt(zmq.RCVHWM, 100000) # Expose 29000 port to database / application clients pubEndpoint = "tcp://127.0.0.1:29000" pubSocket = ctx.socket(zmq.PUB) pubSocket.bind(pubEndpoint) pubSocket.setsockopt(zmq.SNDHWM, 100000) while True: try: msg = pullSocket.recv() decodedMsg = msg.decode('utf-8') msgJson = json.loads(decodedMsg) if args.print: util.log_info(msgJson) pubSocket.send_string(decodedMsg) except KeyboardInterrupt as e: raise except Exception as e: traceback.print_exc() util.log_warning("except caught")
if line_test_name == target_test_name: subtasks.append(line_subtask) return subtasks if __name__ == '__main__': if len(sys.argv) != 3: usage() test_name = sys.argv[1] mapping_file = sys.argv[2] test_subtasks = get_test_subtasks(mapping_file, test_name) if len(test_subtasks) == 0: log_warning("Test '%s' is in no subtasks." % test_name) data = load_json(SUBTASKS_JSON) test_validators = data.get('global_validators', []) if len(test_validators) == 0: log_warning("There is no global validator for the problem.") for subtask in test_subtasks: test_validators += navigate_json(data, 'subtasks/%s' % subtask, SUBTASKS_JSON).get('validators', []) seen = set() for validator in test_validators: if validator not in seen: seen.add(validator)
def check_pm(msgs): """ function for checking bot's private messages; delete if parent commentor (or subreddit moderator) has requested post deletion; stops bot if moderator has requested a halt """ for msg in msgs: # check for delete request m = re.search(ur'^\+delete\s(.+?)$', msg.body.lower()) if m: id = "t1_%s" % m.group(1) c = r.get_info(thing_id = id) if c is not None: c_parent = r.get_info(thing_id = c.parent_id) if c_parent.author is None: delete_post(c, 'PM', '', msg.author.name) else: if msg.author.name == c_parent.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123': if "Please request this post to be deleted to un-ignore." in c.body: log_msg("Un-ignoring posts under %s by %s's request" % (c_parent.permalink, msg.author.name)) update_db(log_coll, stat_coll, 'UNIGNORE', c_parent.permalink, msg.author.name) try: ignored_submissions.remove(str(c.parent_id)[3:]) except Exception as e: log_error(e) delete_post(c, 'PM', c_parent.permalink, msg.author.name) else: log_warning("Incorrect delete request from %s for %s" % (msg.author.name, c.permalink)) update_db(log_coll, stat_coll, 'DEL_BAD', c.permalink, msg.author.name) # check for ignore request m = re.search(ur'^\+ignore\s(.+?)$', msg.body.lower()) if m: id = "t3_%s" % m.group(1) c = r.get_info(thing_id = id) if c.author is not None and (msg.author.name == c.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123'): if m.group(1) not in ignored_submissions: ignored_submissions.append(m.group(1)) log_msg("Ignoring posts under %s by %s's request" % (c.short_link, msg.author.name)) update_db(log_coll, stat_coll, 'IGNORE_PM', c.short_link, msg.author.name) temp_msg = "%s\nI am ignoring any new posts in this thread by OP/moderator's request! Please request this post to be deleted to un-ignore.\n" % signature_intro create_post(c, [temp_msg], 'SUBMISSIONS', 'IGNORE_POST') else: if type(c) is praw.objects.Submission: tempLink = c.short_link elif type(c) is praw.objects.Comment: tempLink = c.permalink else: tempLink = m.group(1) log_warning("Incorrect ignore request from %s for %s" % (msg.author.name, tempLink)) update_db(log_coll, stat_coll, 'IGNORE_BAD', tempLink, msg.author.name) # check for revisit m = re.search(ur'^\+visit\s(.+?)$', msg.body.lower()) if m: temp_type = 'SUBMISSIONS' id = "t3_%s" % m.group(1) c = r.get_info(thing_id = id) if c is None: temp_type = 'COMMENTS' id = id = "t1_%s" % m.group(1) c = r.get_info(thing_id = id) if c is not None and c.subreddit.display_name.lower() == SUBREDDIT.lower() and (msg.author.name == c.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123'): log_msg("Revisiting %s under %s's request" % (c.permalink, msg.author.name)) update_db(log_coll, stat_coll, 'REVISIT', c.permalink, msg.author.name) check_posts([c], temp_type, True) else: if type(c) is praw.objects.Submission: tempLink = c.short_link elif type(c) is praw.objects.Comment: tempLink = c.permalink else: tempLink = m.group(1) log_msg("Incorrect revisit request for %s by %s" % (tempLink, msg.author.name)) update_db(log_coll, stat_coll, 'REVISIT_BAD', tempLink, msg.author.name) # check for moderator halt request if msg.author.name in mod_list or msg.author.name == 'mrmin123': m = re.search(ur'^\+halt$', msg.body.lower()) if m: msg.mark_as_read() log_error("Bot halt requested by %s" % msg.author.name) update_db(log_coll, stat_coll, 'HALT', '', msg.author.name) exit() msg.mark_as_read() sleep(SLEEP) sleep(SLEEP_LONG)