def get_test_validators(test_name, tests_dir):
try:
test_subtasks = tu.get_test_subtasks_from_tests_dir(
test_name, tests_dir)
except tu.MalformedTestsException as e:
sys.stderr.write("{}\n".format(e))
sys.exit(3)
if len(test_subtasks) == 0:
log_warning("Test '%s' is in no subtasks." % test_name)
data = load_json(SUBTASKS_JSON)
global_validators = data.get('global_validators', [])
subtask_sensitive_validators = data.get('subtask_sensitive_validators', [])
def check_subtask_sensitive_validators(subtask_sensitive_validators):
subtask_placeholder_var = "subtask"
subtask_placeholder_test_substitute = "___SUBTASK_PLACEHOLDER_SUBSTITUTE___"
for subtask_sensitive_validator in subtask_sensitive_validators:
try:
subtask_validator_substituted = subtask_sensitive_validator.format(
**{
subtask_placeholder_var:
subtask_placeholder_test_substitute
})
except KeyError as e:
sys.stderr.write(
'Subtask-sensitive validator "{}" contains unknown placeholder {{{}}}.\n'
.format(subtask_sensitive_validator, e.args[0]))
sys.exit(3)
else:
if subtask_placeholder_test_substitute not in subtask_validator_substituted:
log_warning(
'Subtask-sensitive validator "{}" does not contain the subtask placeholder {{{}}}.'
.format(subtask_sensitive_validator,
subtask_placeholder_var))
check_subtask_sensitive_validators(subtask_sensitive_validators)
test_validators = list(global_validators)
for subtask in test_subtasks:
test_validators += [
validator.format(subtask=subtask)
for validator in subtask_sensitive_validators
]
test_validators += navigate_json(data, 'subtasks/%s' % subtask,
SUBTASKS_JSON).get('validators', [])
if len(test_validators) == 0:
log_warning("There is no validator for test {}.".format(test_name))
def unify_list(l):
seen = []
for e in l:
if e not in seen:
seen.append(e)
return seen
return unify_list(test_validators)
def wait_for_disarm(vehicle, timeout=180):
"""
Wait for the vehicle to disarm and log every 2 sec.
Args:
vehicle(dronekit.Vehicle): the copter to be controlled.
timeout(int): seconds before invoking forced exit. Default 30 seconds.
"""
wait_count = 0
sleep_period = 1
log_period = 30
timeout_limit = timeout / sleep_period
log_count = log_period / sleep_period
while vehicle.armed:
if wait_count % log_count == 0: # 10sec period logging
util.log_info("Waiting for the copter to disarm.")
if wait_count >= timeout_limit:
util.log_warning("Wait timeout! Exit script now!")
break
time.sleep(sleep_period)
wait_count = wait_count + 1
def set_mode(vehicle, mode):
"""
Set the vehicle's flight modes. 200ms period state validation.
Args:
vehicle(dronekit.Vehicle): the vehicle to be controlled.
mode(str): flight mode string, supported by the firmware.
Returns:
bool: True if success, False if failed.
Failure will set shared.status['abort'].
"""
util.log_info("Setting %s." % mode)
shared.status['manual_mode'] = mode
vehicle.mode = VehicleMode(mode)
wait_count = 0
while True:
time.sleep(.2)
wait_count = wait_count + 1
if vehicle.mode.name == mode:
return True
elif wait_count >= 45:
util.log_warning("Unable to set %s. Assume link lost." % mode)
shared.status['abort'] = True
return False
elif wait_count % 15 == 0:
util.log_warning("Retry setting %s" % mode)
vehicle.mode = VehicleMode(mode) # resend command
def convert_origin_func_to_npu(node,
origin_func,
org_func_name,
params_list,
is_class_func=None):
"""Convert original Tensorflow function to NPU function"""
if not check_func_arguments(origin_func, node.args, node.keywords,
is_class_func):
return node
if org_func_name == "Estimator.train":
content = "".join([
util_global.get_value('path'), ":",
str(getattr(node, "lineno", "None"))
])
while True:
message = input(
"Check if the train function in " + content +
" is the Estimator train function. If yes, "
"enter 'y' to perform distributed porting on the train function. if no, enter 'n': "
)
if message == "y":
break
if message == "n":
log_warning("".join([
"The train func in ", content,
" is user-defined functions, will not perform distributed porting"
]))
return node
print("Input is error, Please enter 'y' or 'n'.")
for param_name in params_list:
node = match_func_params_and_convert(node, origin_func, org_func_name,
param_name, is_class_func)
util_global.set_value('need_conver', True)
return node
def conver():
"""The entry point to convert Tensorflow script"""
print("Begin conver, input file: " + util_global.get_value('input') + '\n')
out_path = util_global.get_value('output')
dst_path = os.path.split(util_global.get_value('input').rstrip('\\/'))[-1]
dst_path_new = dst_path + util_global.get_value('timestap')
conver_path = os.walk(util_global.get_value('input'))
report_dir = util_global.get_value('report')
mkdir(report_dir)
report_xlsx = os.path.join(report_dir, 'api_analysis_report.xlsx')
util_global.set_value('generate_dir_report', pd.DataFrame())
for path, _, file_list in conver_path:
for file_name in file_list:
out_path_dst = abs_join(
dst_path_new,
path.split(util_global.get_value('input'))[1])
file_path = os.path.join(path, file_name).replace('\\', '/')
if not check_path_length(file_path):
content = "".join([
"The file:", file_path, " length is invalid, skip convert."
])
log_warning(content)
continue
content = "".join(["Begin conver file: ", file_path])
print(content)
threshold_file_size = 10 * 1024 * 1024
if file_name.endswith(".py"):
if os.path.getsize(file_path) > threshold_file_size:
content = "".join([
"The file:", file_path,
" size is over 10M, skip convert."
])
log_warning(content)
continue
util_global.set_value('path', file_path)
mkdir(os.path.join(out_path, out_path_dst))
conver_ast(path, out_path_dst, file_name)
if util_global.get_value('need_conver', False):
content = "".join(
["Finish conver file: ", file_path, '\n'])
print(content)
write_report_terminator(content)
else:
mkdir_and_copyfile(path, abs_join(out_path, out_path_dst),
file_name)
else:
mkdir_and_copyfile(path, abs_join(out_path, out_path_dst),
file_name)
adjust_index()
analysis_report = util_global.get_value('generate_dir_report')
if analysis_report.empty:
print('No api data in the report')
else:
analysis_report.to_excel(report_xlsx, index=True)
get_api_statistic(analysis_report)
print("Finish conver, output file: " + out_path + "; report file: " +
util_global.get_value('report'))
def read_ignore_list(path):
apps = []
if not os.path.exists(path):
log_warning("Can't not find the ignore list at '%s'" % path)
return apps
with io.open(path, 'r', encoding='utf8') as f:
for line in f:
app_id = line.split(' ')[0]
apps.append(int(app_id))
return apps
def convert_tf_gradient_distributed(node):
"""Convert Tensorflow gradient APIs in distributed mode"""
content = "".join([
util_global.get_value('path'), ":",
str(getattr(node, "lineno", "None")),
" is tf.gradient api, tool inserts npu_allreduce after computing grads by default.",
" You can adjust the allreduce position according to the algorithm"
])
log_warning(content)
new_node = ast.Call(func=ast.Name(id="npu_allreduce", ctx=ast.Load()),
args=[node],
keywords=[])
ast.copy_location(new_node, node)
util_global.set_value("need_conver", True)
return new_node
def check_subtask_sensitive_validators(subtask_sensitive_validators):
subtask_placeholder_var = "subtask"
subtask_placeholder_test_substitute = "___SUBTASK_PLACEHOLDER_SUBSTITUTE___"
for subtask_sensitive_validator in subtask_sensitive_validators:
try:
subtask_validator_substituted = subtask_sensitive_validator.format(
**{
subtask_placeholder_var:
subtask_placeholder_test_substitute
})
except KeyError as e:
sys.stderr.write(
'Subtask-sensitive validator "{}" contains unknown placeholder {{{}}}.\n'
.format(subtask_sensitive_validator, e.args[0]))
exit(3)
else:
if subtask_placeholder_test_substitute not in subtask_validator_substituted:
log_warning(
'Subtask-sensitive validator "{}" does not contain the subtask placeholder {{{}}}.'
.format(subtask_sensitive_validator,
subtask_placeholder_var))
def __init__(self, id):
self.REQUESTING = True
self.id = id
self.generated = 0
while self.REQUESTING:
try:
self.html = requests.get("http://www.puzzledragonx.com/en/monster.asp?n=%s" % self.id).content
self.parsed_html = BeautifulSoup(self.html, from_encoding="utf-8")
self.name = self.parsed_html.body.find('h1').text
self.get_data()
self.generated = int(strftime("%Y%m%d", gmtime()))
self.status = 1
self.REQUESTING = False
self.html = None
self.parsed_html = None
if self.name == "Puzzle Dragon X":
self.status = 0
except Exception as e:
log_warning(e)
self.status = 0
sleep(SLEEP_LONG)
def _check_terminate_condition(vehicle, name_string):
"""
Check several termination conditions and set the stopflag to a thread.
Priority:
shared.status['abort'] == True: emergency, highest level.
shared.status['thread_flag'] has FLOCKING_FLAG: thread killing command.
vehicle.mode != 'GUIDED': grabbed manual control.
Args:
vehicle(dronekit.Vehicle): the copter to be controlled.
name_string(str): a string specifies the thread name.
Returns:
bool: True if condition is met, False if no-stop.
"""
status = False
exit_number = 0
if shared.status['abort']:
util.log_warning("Abort detected!")
status = True
exit_number = 1
elif shared.status['thread_flag'] & shared.FLOCKING_FLAG:
util.log_info("FLOCKING_FLAG detected.")
status = True
exit_number = 2
elif vehicle.mode != 'GUIDED':
util.log_warning("Mode switched out from 'GUIDED'!")
status = True
exit_number = 3
if status: util.log_info("Stopping %s" % name_string)
return [status, exit_number]
def arm_and_takeoff(vehicle, target_alt, loiter_time):
"""
Arm the vehicle and takeoff to target altitude.
This is a blocking check, so the program is stuck indefinately unless
received `LIFT` clearance, stopped by `EXIT` command or killed by
KeyboardInterrupt.
Args:
vehicle(dronekit.Vehicle): the copter to be controlled.
target_alt(float): takeoff altitude, validated in the main script.
loiter_time(int): seconds to wait after `des_alt` is reached.
Returns:
bool: True if success, False if killed by `EXIT` command.
"""
util.log_info("Checking if armable.")
wait_count = 0
while not vehicle.is_armable:
time.sleep(.2)
wait_count = wait_count + 1
if wait_count % 25 == 0:
util.log_warning("Vehicle not armable.")
if wait_count >= 100:
util.log_warning("Unable to arm. Abort.")
shared.status['abort'] = True
return False
util.log_info("Switching to GUIDED and Arming.")
set_mode(vehicle, "GUIDED")
util.log_debug("Arming...")
vehicle.armed = True
time.sleep(3)
wait_count = 0
while True:
time.sleep(.5)
wait_count = wait_count + 1
if vehicle.armed:
util.log_info("Armed.")
break
elif wait_count % 10 == 0:
util.log_warning('Retry arming.')
if wait_count >= 20:
util.log_warning("Arming failed. Abort.")
shared.status['abort'] = True
return False
vehicle.simple_takeoff(target_alt)
# Wait until the vehicle reaches a safe altitude (95%), or otherwise the command
# after Vehicle.simple_takeoff will execute immediately, causing unexpected results.
while True:
util.log_debug("Altitude: %s" %
vehicle.location.global_relative_frame.alt)
if vehicle.location.global_relative_frame.alt >= target_alt - shared.ALT_TOLERANCE:
util.log_info("Target altitude reached: %s m." %
vehicle.location.global_relative_frame.alt)
break
time.sleep(.5)
time.sleep(loiter_time)
return True
def _preflight_check(vehicle, xbee):
"""
Perform preflight check, validate home coordinates and satellites fix.
This is a blocking check, so the program is stuck indefinately unless
stopped by `EXIT` command or killed by KeyboardInterrupt.
Args:
vehicle(dronekit.Vehicle): the vehicle to be controlled.
xbee(xbee.Zigbee): the XBee communication interface.
Returns:
bool: True if success, False if killed by `EXIT` command.
"""
util.log_info("Waiting for home location.")
while not vehicle.home_location:
if shared.status['command'] == 'EXIT':
comm.xbee_broadcast(xbee,
"IFO,%s abort takeoff." % shared.AGENT_ID)
util.log_info("'EXIT' received. Abort takeoff.")
return False
time.sleep(2)
cmds = vehicle.commands
cmds.download()
cmds.wait_ready()
comm.xbee_broadcast(xbee, "IFO,%s getting home fix." % shared.AGENT_ID)
# We have a home location now.
comm.xbee_broadcast(
xbee, 'IFO,%s home: %s.' % (shared.AGENT_ID, vehicle.home_location))
util.log_info('Home location: %s' % vehicle.home_location)
# Check satellite condition to ensure 3D-fix first
while vehicle.gps_0.fix_type < 3:
if shared.status['command'] == 'EXIT':
comm.xbee_broadcast(xbee,
"IFO,%s abort takeoff." % shared.AGENT_ID)
util.log_info("'EXIT' received. Abort takeoff.")
return False
comm.xbee_broadcast(xbee, "IFO,%s GNSS No 3D-fix." % shared.AGENT_ID)
util.log_warning("GNSS No 3D Fix.")
time.sleep(3)
# APM:Copter parameter: GPS_HDOP_GOOD
# The value is mutiplied by 100 into a integer, default good HDOP is below 140
while vehicle.gps_0.eph > 140 or vehicle.gps_0.satellites_visible < 9:
if shared.status['command'] == 'EXIT':
comm.xbee_broadcast(xbee,
"IFO,%s abort takeoff." % shared.AGENT_ID)
util.log_info("'EXIT' received. Abort takeoff.")
return False
util.log_info(
"HDOP: %.2f NumSat: %s" %
(vehicle.gps_0.eph / 100.0, vehicle.gps_0.satellites_visible))
comm.xbee_broadcast(
xbee, "IFO,%s HDOP: %.2f NumSat: %s" %
(shared.AGENT_ID, vehicle.gps_0.eph / 100.0,
vehicle.gps_0.satellites_visible))
time.sleep(3)
# --END of while-- Preflight check passed.
comm.xbee_broadcast(xbee,
"IFO,%s Preflight check passed." % shared.AGENT_ID)
util.log_info(
"Preflight check passed. HDOP: %.2f NumSats: %s" %
(vehicle.gps_0.eph / 100.0, vehicle.gps_0.satellites_visible))
util.log_info("Local time %s" % shared.timestamp)
return True
for (app_id, app_name) in apps:
count += 1
file_path = meta_dir_path + "/app_meta_" + str(app_id) + ".json"
# Check if the metadata exists
if (not args.re_meta) and os.path.exists(file_path):
continue
# Downloads the json data
data = retrieve_data(APP_URL_PREFIX + str(app_id) + "&cc=us")
if data is None:
continue
if data[str(app_id)]["success"] is False:
log_warning("Failed to retrieve the data for '%s' (id: %d)" % (app_name, app_id))
continue
data = data[str(app_id)]["data"]
# Only keep the attributes we care about
new_data = {}
for attr in APP_ATTRS:
if attr in data:
new_data[attr] = data[attr]
# Save to the file
with open(file_path, "w") as f:
f.write(json.dumps(new_data))
down_count += 1
if down_count % 10 == 0:
help="print the logs")
args = parser.parse_args()
# Connect with IPC fed by exchange_md.py
ctx = zmq.Context()
pullEndpoint = "ipc:///home/ubuntu/tmp/.md_server_private_zmq_address"
pullSocket = ctx.socket(zmq.PULL)
pullSocket.bind(pullEndpoint)
pullSocket.setsockopt(zmq.RCVHWM, 100000)
# Expose 29000 port to database / application clients
pubEndpoint = "tcp://127.0.0.1:29000"
pubSocket = ctx.socket(zmq.PUB)
pubSocket.bind(pubEndpoint)
pubSocket.setsockopt(zmq.SNDHWM, 100000)
while True:
try:
msg = pullSocket.recv()
decodedMsg = msg.decode('utf-8')
msgJson = json.loads(decodedMsg)
if args.print:
util.log_info(msgJson)
pubSocket.send_string(decodedMsg)
except KeyboardInterrupt as e:
raise
except Exception as e:
traceback.print_exc()
util.log_warning("except caught")
if line_test_name == target_test_name:
subtasks.append(line_subtask)
return subtasks
if __name__ == '__main__':
if len(sys.argv) != 3:
usage()
test_name = sys.argv[1]
mapping_file = sys.argv[2]
test_subtasks = get_test_subtasks(mapping_file, test_name)
if len(test_subtasks) == 0:
log_warning("Test '%s' is in no subtasks." % test_name)
data = load_json(SUBTASKS_JSON)
test_validators = data.get('global_validators', [])
if len(test_validators) == 0:
log_warning("There is no global validator for the problem.")
for subtask in test_subtasks:
test_validators += navigate_json(data, 'subtasks/%s' % subtask,
SUBTASKS_JSON).get('validators', [])
seen = set()
for validator in test_validators:
if validator not in seen:
seen.add(validator)
def check_pm(msgs):
"""
function for checking bot's private messages; delete if parent commentor
(or subreddit moderator) has requested post deletion; stops bot if
moderator has requested a halt
"""
for msg in msgs:
# check for delete request
m = re.search(ur'^\+delete\s(.+?)$', msg.body.lower())
if m:
id = "t1_%s" % m.group(1)
c = r.get_info(thing_id = id)
if c is not None:
c_parent = r.get_info(thing_id = c.parent_id)
if c_parent.author is None:
delete_post(c, 'PM', '', msg.author.name)
else:
if msg.author.name == c_parent.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123':
if "Please request this post to be deleted to un-ignore." in c.body:
log_msg("Un-ignoring posts under %s by %s's request" % (c_parent.permalink, msg.author.name))
update_db(log_coll, stat_coll, 'UNIGNORE', c_parent.permalink, msg.author.name)
try:
ignored_submissions.remove(str(c.parent_id)[3:])
except Exception as e:
log_error(e)
delete_post(c, 'PM', c_parent.permalink, msg.author.name)
else:
log_warning("Incorrect delete request from %s for %s" % (msg.author.name, c.permalink))
update_db(log_coll, stat_coll, 'DEL_BAD', c.permalink, msg.author.name)
# check for ignore request
m = re.search(ur'^\+ignore\s(.+?)$', msg.body.lower())
if m:
id = "t3_%s" % m.group(1)
c = r.get_info(thing_id = id)
if c.author is not None and (msg.author.name == c.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123'):
if m.group(1) not in ignored_submissions:
ignored_submissions.append(m.group(1))
log_msg("Ignoring posts under %s by %s's request" % (c.short_link, msg.author.name))
update_db(log_coll, stat_coll, 'IGNORE_PM', c.short_link, msg.author.name)
temp_msg = "%s\nI am ignoring any new posts in this thread by OP/moderator's request! Please request this post to be deleted to un-ignore.\n" % signature_intro
create_post(c, [temp_msg], 'SUBMISSIONS', 'IGNORE_POST')
else:
if type(c) is praw.objects.Submission:
tempLink = c.short_link
elif type(c) is praw.objects.Comment:
tempLink = c.permalink
else:
tempLink = m.group(1)
log_warning("Incorrect ignore request from %s for %s" % (msg.author.name, tempLink))
update_db(log_coll, stat_coll, 'IGNORE_BAD', tempLink, msg.author.name)
# check for revisit
m = re.search(ur'^\+visit\s(.+?)$', msg.body.lower())
if m:
temp_type = 'SUBMISSIONS'
id = "t3_%s" % m.group(1)
c = r.get_info(thing_id = id)
if c is None:
temp_type = 'COMMENTS'
id = id = "t1_%s" % m.group(1)
c = r.get_info(thing_id = id)
if c is not None and c.subreddit.display_name.lower() == SUBREDDIT.lower() and (msg.author.name == c.author.name or msg.author.name in mod_list or msg.author.name == 'mrmin123'):
log_msg("Revisiting %s under %s's request" % (c.permalink, msg.author.name))
update_db(log_coll, stat_coll, 'REVISIT', c.permalink, msg.author.name)
check_posts([c], temp_type, True)
else:
if type(c) is praw.objects.Submission:
tempLink = c.short_link
elif type(c) is praw.objects.Comment:
tempLink = c.permalink
else:
tempLink = m.group(1)
log_msg("Incorrect revisit request for %s by %s" % (tempLink, msg.author.name))
update_db(log_coll, stat_coll, 'REVISIT_BAD', tempLink, msg.author.name)
# check for moderator halt request
if msg.author.name in mod_list or msg.author.name == 'mrmin123':
m = re.search(ur'^\+halt$', msg.body.lower())
if m:
msg.mark_as_read()
log_error("Bot halt requested by %s" % msg.author.name)
update_db(log_coll, stat_coll, 'HALT', '', msg.author.name)
exit()
msg.mark_as_read()
sleep(SLEEP)
sleep(SLEEP_LONG)