def get_marketId():
log_info('get_marketId')
def chk(result):
entries = {}
for(success, r) in result:
if not success:
log_error('create db table failed {}'.format(r))
else:
prodID, hubID, d = r
log_info('prodID {}, hubID {}'.format(prodID, hubID))
for i in d:
i['productId'] = prodID
i['hubId'] = hubID
i['marketId'] = int(i['marketId'])
i['change'] = int(i['change'])
i['endDate'] = to_utctimestamp(i['endDate'])
i['lastTime'] = to_utctimestamp(i['lastTime'])
entries[i['marketId']] = i
log_info('num of marketID downloaded: {}'.format(len(entries)))
return entries
dl = []
for prodID, hubID in contract_list:
url = f'https://www.theice.com/marketdata/DelayedMarkets.shtml?getContractsAsJson=&productId={prodID}&hubId={hubID}'
log_info(url)
d = download_contracts(url, prodID, hubID)
dl.append(d)
dlist = defer.DeferredList(dl)
dlist.addCallback(chk)
return dlist
def add_disassembly(self, filter_lib):
""" Collect disassembly information:
1. Use objdump to collect disassembly for each function in FunctionSet.
2. Set flag to dump addr_hit_map when generating record info.
"""
objdump = Objdump(self.ndk_path, self.binary_cache_path)
cur_lib_name = None
dso_info = None
for function in sorted(self.functions.id_to_func.values(),
key=lambda a: a.lib_id):
if function.func_name == 'unknown':
continue
lib_name = self.libs.get_lib_name(function.lib_id)
if lib_name != cur_lib_name:
cur_lib_name = lib_name
if filter_lib(lib_name):
dso_info = objdump.get_dso_info(lib_name)
else:
dso_info = None
if dso_info:
log_info('Disassemble %s' % dso_info[0])
if dso_info:
code = objdump.disassemble_code(dso_info, function.start_addr,
function.addr_len)
function.disassembly = code
self.gen_addr_hit_map_in_record_info = True
def decode(self):
"""
Verifies that the receipt can be decoded and that the initial
contents of the receipt are correct.
If its invalid, then just return invalid rather than give out any
information.
"""
try:
receipt = decode_receipt(self.receipt)
except:
log_exception({
'receipt': '%s...' % self.receipt[:10],
'app': self.get_app_id(raise_exception=False)
})
log_info('Error decoding receipt')
raise InvalidReceipt('ERROR_DECODING')
try:
assert receipt['user']['type'] == 'directed-identifier'
except (AssertionError, KeyError):
log_info('No directed-identifier supplied')
raise InvalidReceipt('NO_DIRECTED_IDENTIFIER')
return receipt
def tba_request(api_url):
"""Sends a single web request to the TBA API v3 api_url is the suffix of the API request URL
(the part after '/api/v3').
"""
utils.log_info(f'tba request from {api_url} started')
full_url = f'https://www.thebluealliance.com/api/v3/{api_url}'
request_headers = {'X-TBA-Auth-Key': API_KEY}
cached = local_database_communicator.select_tba_cache(api_url)
# Check if cache exists
if cached != {}:
cached = cached[api_url]
request_headers['If-Modified-Since'] = cached['timestamp']
print(f'Retrieving data from {full_url}')
utils.log_info(f'tba request from {api_url} finished')
try:
request = requests.get(full_url, headers=request_headers)
except requests.exceptions.ConnectionError:
utils.log_warning('Error: No internet connection.')
return None
# A 200 status code means the request was successful
# 304 means that data was not modified since the last timestamp
# specified in request_headers['If-Modified-Since']
if request.status_code == 304:
return cached['data']
if request.status_code == 200:
formatted_data = {
'timestamp': request.headers['Last-Modified'],
'data': request.json()
}
local_database_communicator.overwrite_tba_data(formatted_data, api_url)
return request.json()
raise Warning(f'Request failed with status code {request.status_code}')
def _second_parse(self):
log_info('second_parse begin')
parser = self.parser
while parser.has_more_commands():
line = ''
parser.advance()
command_type = parser.command_type()
if command_type == COMMAND.A_COMMAND:
symbol = parser.symbol()
# log_debug("A_COMMAND, symbol - {}".format(symbol))
address = self._parse_address(symbol)
line = "0{:015b}\n".format(address)
elif command_type == COMMAND.L_COMMAND:
symbol = parser.symbol()
# log_debug("L_COMMAND, symbol - {}".format(symbol))
elif command_type == COMMAND.C_COMMAND:
# dest_str = parser.dest()
# comp_str = parser.comp()
# jump_str = parser.jump()
dest_str, comp_str, jump_str = parser.parse_c_command()
# log_debug("C_COMMAND, dest - {!r}, comp - {!r}, jump - {!r}".format(dest_str, comp_str, jump_str))
dest = self.code.dest(dest_str)
comp = self.code.comp(comp_str)
jump = self.code.jump(jump_str)
line = '111{:07b}{:03b}{:03b}\n'.format(comp, dest, jump)
else:
raise Exception("Unexpected command - {}".format(command_type))
# log_debug("line - {}".format(line))
if line != '':
self.output.append(line)
log_info('second_parse end')
def export_tim_data(path):
"""Takes team in match data and writes to CSV. Puts team in match export files into their own
directory to separate them from team export files.
"""
# Get the lists of column headers and dictionaries to use in export
all_tim_data = load_data(TIM_DATA_DB_PATHS)
column_headers = format_header(all_tim_data, ['match_number', 'team_number'])
with open(path, 'w') as file:
# Write headers using the column_headers list
csv_writer = csv.DictWriter(file, fieldnames=column_headers)
csv_writer.writeheader()
for team in TEAMS_LIST:
# Write rows using data in dictionary
team_data = []
for document in all_tim_data:
if document['team_number'] == team:
team_data.append(document)
tim_data = {}
for document in team_data:
if document['match_number'] in tim_data:
tim_data[document['match_number']].update(document)
else:
tim_data[document['match_number']] = document
for document in tim_data.values():
csv_writer.writerow(document)
utils.log_info('Exported TIM to CSV')
def add_disassembly(self, filter_lib):
""" Collect disassembly information:
1. Use objdump to collect disassembly for each function in FunctionSet.
2. Set flag to dump addr_hit_map when generating record info.
"""
objdump = Objdump(self.ndk_path, self.binary_cache_path)
cur_lib_name = None
dso_info = None
for function in sorted(self.functions.id_to_func.values(), key=lambda a: a.lib_id):
if function.func_name == 'unknown':
continue
lib_name = self.libs.get_lib_name(function.lib_id)
if lib_name != cur_lib_name:
cur_lib_name = lib_name
if filter_lib(lib_name):
dso_info = objdump.get_dso_info(lib_name)
else:
dso_info = None
if dso_info:
log_info('Disassemble %s' % dso_info[0])
if dso_info:
code = objdump.disassemble_code(dso_info, function.start_addr, function.addr_len)
function.disassembly = code
self.gen_addr_hit_map_in_record_info = True
def check_db(self):
"""
Verifies the decoded receipt against the database.
Requires that decode is run first.
"""
if not self.decoded:
raise ValueError('decode not run')
self.setup_db()
# Get the addon and user information from the installed table.
try:
self.uuid = self.decoded['user']['value']
except KeyError:
# If somehow we got a valid receipt without a uuid
# that's a problem. Log here.
log_info('No user in receipt')
raise InvalidReceipt('NO_USER')
try:
storedata = self.decoded['product']['storedata']
self.addon_id = int(dict(parse_qsl(storedata)).get('id', ''))
except:
# There was some value for storedata but it was invalid.
log_info('Invalid store data')
raise InvalidReceipt('WRONG_STOREDATA')
def decompress_single_qr(qr_data, qr_type):
"""Decompress a full QR."""
# Split into generic data and objective/subjective data
qr_data = qr_data.split(SCHEMA['generic_data']['_section_separator'])
# Generic QR is first section of QR
decompressed_data = decompress_generic_qr(qr_data[0])
# Decompress subjective QR
if qr_type == QRType.SUBJECTIVE:
subjective_data = qr_data[1].split(
SCHEMA['subjective_aim']['_separator'])
decompressed_data.update(
decompress_data(subjective_data, 'subjective_aim'))
if set(decompressed_data.keys()) != SUBJECTIVE_QR_FIELDS:
raise ValueError('QR missing data fields')
elif qr_type == QRType.OBJECTIVE: # Decompress objective QR
objective_data = qr_data[1].split(
SCHEMA['objective_tim']['_separator'])
decompressed_data.update(
decompress_data(objective_data, 'objective_tim'))
if set(decompressed_data.keys()) != OBJECTIVE_QR_FIELDS:
raise ValueError('QR missing data fields')
utils.log_info(f'Match: {decompressed_data["match_number"]} '
f'Team: {decompressed_data["team_number"]} '
f'Scout_ID: {decompressed_data["scout_id"]}')
return decompressed_data
def collect_data(args):
""" Run app_profiler.py to generate record file. """
app_profiler_args = [sys.executable, os.path.join(scripts_path, "app_profiler.py"), "-nb"]
if args.app:
app_profiler_args += ["-p", args.app]
elif args.native_program:
app_profiler_args += ["-np", args.native_program]
else:
log_exit("Please set profiling target with -p or -np option.")
if args.compile_java_code:
app_profiler_args.append("--compile_java_code")
if args.disable_adb_root:
app_profiler_args.append("--disable_adb_root")
record_arg_str = ""
if args.dwarf_unwinding:
record_arg_str += "-g "
else:
record_arg_str += "--call-graph fp "
if args.events:
tokens = args.events.split()
if len(tokens) == 2:
num_events = tokens[0]
event_name = tokens[1]
record_arg_str += "-c %s -e %s " % (num_events, event_name)
else:
log_exit("Event format string of -e option cann't be recognized.")
log_info("Using event sampling (-c %s -e %s)." % (num_events, event_name))
else:
record_arg_str += "-f %d " % args.sample_frequency
log_info("Using frequency sampling (-f %d)." % args.sample_frequency)
record_arg_str += "--duration %d " % args.capture_duration
app_profiler_args += ["-r", record_arg_str]
returncode = subprocess.call(app_profiler_args)
return returncode == 0
def parse_samples(process, args, sample_filter_fn):
"""Read samples from record file.
process: Process object
args: arguments
sample_filter_fn: if not None, is used to modify and filter samples.
It returns false for samples should be filtered out.
"""
record_file = args.record_file
symfs_dir = args.symfs
kallsyms_file = args.kallsyms
lib = ReportLib()
lib.ShowIpForUnknownSymbol()
if symfs_dir:
lib.SetSymfs(symfs_dir)
if record_file:
lib.SetRecordFile(record_file)
if kallsyms_file:
lib.SetKallsymsFile(kallsyms_file)
if args.show_art_frames:
lib.ShowArtFrames(True)
process.cmd = lib.GetRecordCmd()
product_props = lib.MetaInfo().get("product_props")
if product_props:
manufacturer, model, name = product_props.split(':')
process.props['ro.product.manufacturer'] = manufacturer
process.props['ro.product.model'] = model
process.props['ro.product.name'] = name
if lib.MetaInfo().get('trace_offcpu') == 'true':
process.props['trace_offcpu'] = True
if args.one_flamegraph:
log_exit("It doesn't make sense to report with --one-flamegraph for perf.data " +
"recorded with --trace-offcpu.""")
else:
process.props['trace_offcpu'] = False
while True:
sample = lib.GetNextSample()
if sample is None:
lib.Close()
break
symbol = lib.GetSymbolOfCurrentSample()
callchain = lib.GetCallChainOfCurrentSample()
if sample_filter_fn and not sample_filter_fn(sample, symbol, callchain):
continue
process.add_sample(sample, symbol, callchain)
if process.pid == 0:
main_threads = [thread for thread in process.threads.values() if thread.tid == thread.pid]
if main_threads:
process.name = main_threads[0].name
process.pid = main_threads[0].pid
for thread in process.threads.values():
min_event_count = thread.num_events * args.min_callchain_percentage * 0.01
thread.flamegraph.trim_callchain(min_event_count)
log_info("Parsed %s callchains." % process.num_samples)
def application(environ, start_response):
status = '200 OK'
with statsd.timer('services.verify'):
data = environ['wsgi.input'].read()
try:
addon_id = id_re.search(environ['PATH_INFO']).group('addon_id')
except AttributeError:
output = ''
log_info({'receipt': '%s...' % data[:10], 'addon': 'empty'},
'Wrong url %s' % environ['PATH_INFO'][:20])
start_response('500 Internal Server Error', [])
return [output]
try:
verify = Verify(addon_id, data, environ)
output = verify()
start_response(status, verify.get_headers(len(output)))
receipt_cef.log(environ, addon_id, 'verify',
'Receipt verification')
except:
output = ''
log_exception({'receipt': '%s...' % data[:10], 'addon': addon_id})
receipt_cef.log(environ, addon_id, 'verify',
'Receipt verification error')
start_response('500 Internal Server Error', [])
return [output]
def main():
parser = argparse.ArgumentParser(description="""
Annotate source files based on profiling data. It reads line information from binary_cache
generated by app_profiler.py or binary_cache_builder.py, and generate annotated source
files in annotated_files directory.""")
parser.add_argument('-i', '--perf_data_list', nargs='+', action='append', help="""
The paths of profiling data. Default is perf.data.""")
parser.add_argument('-s', '--source_dirs', type=extant_dir, nargs='+', action='append', help="""
Directories to find source files.""")
parser.add_argument('--comm', nargs='+', action='append', help="""
Use samples only in threads with selected names.""")
parser.add_argument('--pid', nargs='+', action='append', help="""
Use samples only in processes with selected process ids.""")
parser.add_argument('--tid', nargs='+', action='append', help="""
Use samples only in threads with selected thread ids.""")
parser.add_argument('--dso', nargs='+', action='append', help="""
Use samples only in selected binaries.""")
parser.add_argument('--ndk_path', type=extant_dir, help='Set the path of a ndk release.')
args = parser.parse_args()
config = {}
config['perf_data_list'] = flatten_arg_list(args.perf_data_list)
if not config['perf_data_list']:
config['perf_data_list'].append('perf.data')
config['source_dirs'] = flatten_arg_list(args.source_dirs)
config['comm_filters'] = flatten_arg_list(args.comm)
config['pid_filters'] = flatten_arg_list(args.pid)
config['tid_filters'] = flatten_arg_list(args.tid)
config['dso_filters'] = flatten_arg_list(args.dso)
config['ndk_path'] = args.ndk_path
annotator = SourceFileAnnotator(config)
annotator.annotate()
log_info('annotate finish successfully, please check result in annotated_files/.')
def mint_new_block_and_mine(self):
if not self.mining_paused:
# Put transaction from waiting list into block
try:
chosen_transactions = self.transactions_pool[
0:min(self.block_limit, len(self.transactions_pool))]
if type(chosen_transactions) == type(Transaction):
chosen_transactions = [chosen_transactions]
except IndexError:
chosen_transactions = []
# # Create the block
prev_block = self.chain[[-1]][0]
new_block = Block(previous_block_hash=prev_block.block_hash,
transactions=[
t.export_transaction_to_dict()
for t in chosen_transactions
],
height=prev_block.height + 1,
start_nounce=self.chain_id * NOUNCE_DISTANCE)
log_info('[node..Blockchain.mint_new_block_and_mine] Minted block has ({}) transactions. Mining...'\
.format(len(chosen_transactions)))
# dynamic_log_level.set_log_level(0)
new_block.mine()
# dynamic_log_level.reset_user_log_level()
log_info(
'[node..Blockchain.mint_new_block_and_mine] Block mined @ {}'.
format(new_block.block_hash))
return new_block
def run_checks(file, data_dict, log, commit_range, checks):
"""Checks rule compliance for any given dataset file."""
committed_lines = get_committed_lines(file, commit_range)
uncommitted_lines = get_uncommitted_lines(file)
with open(file, newline="") as csvfile:
info = csv.DictReader(csvfile, data_dict["columns"])
header = next(info)
if "1" in uncommitted_lines or "1" in committed_lines:
check_header(list(header.values()), data_dict, file, log)
if uncommitted_lines != [] or committed_lines != []:
for i, row in enumerate(info):
i += 2
line = str(i)
# The line is either:
# (1) only uncommitted (needs to always bechecked locally),
# (2) only committed (needs to always be checked in CI) or
# (3) both in the unpushed commits and uncommitted (which in
# practice is the same as (1)--the committed one is
# deprecated--)
if (line in uncommitted_lines) or (line in committed_lines):
params = [file, row, line, log]
for check_rule in checks:
if check_rule.__name__ == check_row_length.__name__:
check_rule(len(header), *params)
continue
check_rule(*params)
else:
log_info(file, log, "There are no changes to be checked")
with open(file, 'rb') as fp:
for line in fp:
if line.endswith(b'\r\n'):
log_esp_error(file, log, "Incorrect End of Line encoding")
break
def check_type(self, *types):
"""
Verifies that the type of receipt is what we expect.
"""
if self.decoded.get('typ', '') not in types:
log_info('Receipt type not in %s' % ','.join(types))
raise InvalidReceipt('WRONG_TYPE')
def decode(self):
"""
Verifies that the receipt can be decoded and that the initial
contents of the receipt are correct.
If its invalid, then just return invalid rather than give out any
information.
"""
try:
receipt = decode_receipt(self.receipt)
except:
log_exception({
'receipt': '%s...' % self.receipt[:10],
'addon': self.addon_id
})
log_info('Error decoding receipt')
raise InvalidReceipt
try:
assert receipt['user']['type'] == 'directed-identifier'
except (AssertionError, KeyError):
log_info('No directed-identifier supplied')
raise InvalidReceipt
return receipt
def check_db(self):
"""
Verifies the decoded receipt against the database.
Requires that decode is run first.
"""
if not self.decoded:
raise ValueError('decode not run')
self.setup_db()
# Get the addon and user information from the installed table.
try:
self.uuid = self.decoded['user']['value']
except KeyError:
# If somehow we got a valid receipt without a uuid
# that's a problem. Log here.
log_info('No user in receipt')
raise InvalidReceipt('NO_USER')
try:
storedata = self.decoded['product']['storedata']
self.addon_id = int(dict(parse_qsl(storedata)).get('id', ''))
except:
# There was some value for storedata but it was invalid.
log_info('Invalid store data')
raise InvalidReceipt('WRONG_STOREDATA')
def check_full(self):
"""
This is the default that verify will use, this will
do the entire stack of checks.
"""
receipt_domain = urlparse(settings.WEBAPPS_RECEIPT_URL).netloc
try:
self.decoded = self.decode()
self.check_type('purchase-receipt')
self.check_db()
self.check_url(receipt_domain)
except InvalidReceipt:
return self.invalid()
if self.premium != ADDON_PREMIUM:
log_info('Valid receipt, not premium')
return self.ok_or_expired()
try:
self.check_purchase()
except InvalidReceipt:
return self.invalid()
except RefundedReceipt:
return self.refund()
return self.ok_or_expired()
def export_team_data(path):
"""Takes data team data and writes to CSV.
Merges raw and processed team data into one dictionary
Puts team export files into their own directory
to separate them from team in match export files.
"""
# Get the lists of column headers and dictionaries to use in export
team_data = load_data(TEAM_DATA_DB_PATHS)
column_headers = format_header(team_data, ['team_number'])
# The list of teams, used to merge raw and processed team data
with open(path, 'w') as file:
# Write headers using the column_headers list
csv_writer = csv.DictWriter(file, fieldnames=column_headers)
csv_writer.writeheader()
for team in TEAMS_LIST:
# The dictionary that will hold the combined team data, reset for each team
merged_team = {}
# Go through all dictionaries and check if their team number matches the team's
for document in team_data:
if document.get('team_number') == team:
# Update data from the same team to the merged_team dict
merged_team.update(document)
# Use each team's merged data to write a row
csv_writer.writerow(merged_team)
utils.log_info('Exported team data to CSV')
def application(environ, start_response):
status = "200 OK"
with statsd.timer("services.verify"):
data = environ["wsgi.input"].read()
try:
addon_id = id_re.search(environ["PATH_INFO"]).group("addon_id")
except AttributeError:
output = ""
log_info({"receipt": "%s..." % data[:10], "addon": "empty"}, "Wrong url %s" % environ["PATH_INFO"][:20])
start_response("500 Internal Server Error", [])
return [output]
try:
verify = Verify(addon_id, data, environ)
output = verify()
start_response(status, verify.get_headers(len(output)))
receipt_cef.log(environ, addon_id, "verify", "Receipt verification")
except:
output = ""
log_exception({"receipt": "%s..." % data[:10], "addon": addon_id})
receipt_cef.log(environ, addon_id, "verify", "Receipt verification error")
start_response("500 Internal Server Error", [])
return [output]
def application(environ, start_response):
status = '200 OK'
with statsd.timer('services.verify'):
data = environ['wsgi.input'].read()
try:
addon_id = id_re.search(environ['PATH_INFO']).group('addon_id')
except AttributeError:
output = ''
log_info({
'receipt': '%s...' % data[:10],
'addon': 'empty'
}, 'Wrong url %s' % environ['PATH_INFO'][:20])
start_response('500 Internal Server Error', [])
return [output]
try:
verify = Verify(addon_id, data, environ)
output = verify()
start_response(status, verify.get_headers(len(output)))
receipt_cef.log(environ, addon_id, 'verify',
'Receipt verification')
except:
output = ''
log_exception({'receipt': '%s...' % data[:10], 'addon': addon_id})
receipt_cef.log(environ, addon_id, 'verify',
'Receipt verification error')
start_response('500 Internal Server Error', [])
return [output]
def check_full(self):
"""
This is the default that verify will use, this will
do the entire stack of checks.
"""
try:
self.decoded = self.decode()
self.check_type('purchase-receipt')
self.check_db()
self.check_url()
except InvalidReceipt:
return self.invalid()
if self.premium != ADDON_PREMIUM:
log_info('Valid receipt, not premium')
return self.ok_or_expired()
try:
self.check_purchase()
except InvalidReceipt:
return self.invalid()
except RefundedReceipt:
return self.refund()
return self.ok_or_expired()
def persist_marketId(self, data):
log_info('persist_marketId')
if self.marketIdData == data:
return
else:
self.marketIdData = data
def cb(conn):
records = []
for entry in data.values():
records.append('''("{}", "{}", "{}", "{}", "{}", "{}", "{}", "{}", "{}")'''.format(
entry['productId'],
entry['hubId'],
entry['marketId'],
entry['lastTime'],
entry['endDate'],
entry['lastPrice'],
entry['marketStrip'],
entry['volume'],
entry['change'],
))
v = ','.join(records)
cmd = 'INSERT OR IGNORE INTO Contracts(productId, hubId, marketId, lastTime, endDate, lastPrice, marketStrip, volume, change) values {}'.format(v)
d = conn.runOperation(cmd)
def eb(e):
log_error('Persist data failed {}'.format(e))
log_error(cmd)
d.addErrback(eb)
return d
d = defer.maybeDeferred(self.load)
d.addCallback(cb)
return d
def check_type(self, *types):
"""
Verifies that the type of receipt is what we expect.
"""
if self.decoded.get('typ', '') not in types:
log_info('Receipt type not in %s' % ','.join(types))
raise InvalidReceipt('WRONG_TYPE')
def main():
parser = argparse.ArgumentParser(description="""
Annotate source files based on profiling data. It reads line information from binary_cache
generated by app_profiler.py or binary_cache_builder.py, and generate annotated source
files in annotated_files directory.""")
parser.add_argument('-i',
'--perf_data_list',
nargs='+',
action='append',
help="""
The paths of profiling data. Default is perf.data.""")
parser.add_argument('-s',
'--source_dirs',
type=extant_dir,
nargs='+',
action='append',
help="""
Directories to find source files.""")
parser.add_argument('--comm',
nargs='+',
action='append',
help="""
Use samples only in threads with selected names.""")
parser.add_argument('--pid',
nargs='+',
action='append',
help="""
Use samples only in processes with selected process ids.""")
parser.add_argument('--tid',
nargs='+',
action='append',
help="""
Use samples only in threads with selected thread ids.""")
parser.add_argument('--dso',
nargs='+',
action='append',
help="""
Use samples only in selected binaries.""")
parser.add_argument('--ndk_path',
type=extant_dir,
help='Set the path of a ndk release.')
args = parser.parse_args()
config = {}
config['perf_data_list'] = flatten_arg_list(args.perf_data_list)
if not config['perf_data_list']:
config['perf_data_list'].append('perf.data')
config['source_dirs'] = flatten_arg_list(args.source_dirs)
config['comm_filters'] = flatten_arg_list(args.comm)
config['pid_filters'] = flatten_arg_list(args.pid)
config['tid_filters'] = flatten_arg_list(args.tid)
config['dso_filters'] = flatten_arg_list(args.dso)
config['ndk_path'] = args.ndk_path
annotator = SourceFileAnnotator(config)
annotator.annotate()
log_info(
'annotate finish successfully, please check result in annotated_files/.'
)
def disconnect(self):
ret_val = self.sem_api.ClosingControl()
if ret_val == 0:
utils.log_info('SEM', 'Disconnected from SmartSEM.')
return True
else:
utils.log_error('SEM', f'ERROR disconnecting from SmartSEM (ret_val: {ret_val}).')
return False
def SCHED_mine_for_block(blockchain, sched):
random_id = random.randint(0, 1000)
log_info(
"[SCHED_mine_for_block]({}) Starting routine... ".format(random_id))
return {
'sched': sched,
'blockchain': blockchain,
'new_block': blockchain.mint_new_block_and_mine()
}
def unzip_recording_data(args):
zip_file_path = os.path.join(args.out_dir, 'simpleperf_data.zip')
with zipfile.ZipFile(zip_file_path, 'r') as zip_fh:
names = zip_fh.namelist()
log_info('There are %d recording data files.' % len(names))
for name in names:
log_info('recording file: %s' % os.path.join(args.out_dir, name))
zip_fh.extract(name, args.out_dir)
remove(zip_file_path)
def unzip_recording_data(args):
zip_file_path = os.path.join(args.out_dir, 'simpleperf_data.zip')
with zipfile.ZipFile(zip_file_path, 'r') as zip_fh:
names = zip_fh.namelist()
log_info('There are %d recording data files.' % len(names))
for name in names:
log_info('recording file: %s' % os.path.join(args.out_dir, name))
zip_fh.extract(name, args.out_dir)
remove(zip_file_path)
def gen_source_lines(self):
# 1. Create Addr2line instance
if not self.config.get('binary_cache_dir'):
log_info("Can't generate line information because binary_cache is missing.")
return
if not find_tool_path('addr2line', self.config['ndk_path']):
log_info("Can't generate line information because can't find addr2line.")
return
addr2line = Addr2Nearestline(self.config['ndk_path'], self.config['binary_cache_dir'], True)
# 2. Put all needed addresses to it.
for location in self.location_list:
mapping = self.get_mapping(location.mapping_id)
dso_name = self.get_string(mapping.filename_id)
if location.lines:
function = self.get_function(location.lines[0].function_id)
addr2line.add_addr(dso_name, function.vaddr_in_dso, location.vaddr_in_dso)
for function in self.function_list:
dso_name = self.get_string(function.dso_name_id)
addr2line.add_addr(dso_name, function.vaddr_in_dso, function.vaddr_in_dso)
# 3. Generate source lines.
addr2line.convert_addrs_to_lines()
# 4. Annotate locations and functions.
for location in self.location_list:
if not location.lines:
continue
mapping = self.get_mapping(location.mapping_id)
dso_name = self.get_string(mapping.filename_id)
dso = addr2line.get_dso(dso_name)
if not dso:
continue
sources = addr2line.get_addr_source(dso, location.vaddr_in_dso)
if not sources:
continue
for (source_id, source) in enumerate(sources):
source_file, source_line, function_name = source
function_id = self.get_function_id(function_name, dso_name, 0)
if function_id == 0:
continue
if source_id == 0:
# Clear default line info
location.lines = []
location.lines.append(self.add_line(source_file, source_line, function_id))
for function in self.function_list:
dso_name = self.get_string(function.dso_name_id)
if function.vaddr_in_dso:
dso = addr2line.get_dso(dso_name)
if not dso:
continue
sources = addr2line.get_addr_source(dso, function.vaddr_in_dso)
if sources:
source_file, source_line, _ = sources[0]
function.source_filename_id = self.get_string_id(source_file)
function.start_line = source_line
def make_wallets(amount):
data = []
for i in range(amount):
progress(i,amount-1,'making wallets...')
pub, pri = ppk_keygen()
wallet = {'public':pub,\
'private':pri}
data.append(wallet)
log_info('[client.Client.make_wallets] writing ({}) wallets to pickle...'.format(amount))
utils.save_pickle(data,WALLETS_DIR)
def log_info_perf(self, epoch):
test_perf = self.frac_err(self.W, self.test_images, self.test_labels)
train_perf = self.frac_err(self.W, self.train_images,
self.train_labels)
if test_perf > self.prev_test_perf:
self.learning_rate = 0.1 * self.learning_rate
self.prev_test_perf = test_perf
utils.log_info(
"Epoch {0}, TrainErr {1:5}, TestErr {2:5}, LR {3:2}".format(
self.epoch, train_perf, test_perf, self.learning_rate))
def get_storedata(self):
"""
Attempt to retrieve the storedata information from the receipt.
"""
try:
storedata = self.decoded['product']['storedata']
return dict(parse_qsl(storedata))
except Exception, e:
log_info('Invalid store data: {err}'.format(err=e))
raise InvalidReceipt('WRONG_STOREDATA')
def get_storedata(self):
"""
Attempt to retrieve the storedata information from the receipt.
"""
try:
storedata = self.decoded['product']['storedata']
return dict(parse_qsl(storedata))
except Exception, e:
log_info('Invalid store data: {err}'.format(err=e))
raise InvalidReceipt('WRONG_STOREDATA')
def delete_tablet_downloads():
"""Deletes all data from the Download folder of tablets"""
devices = get_attached_devices()
# Wait for USB connection to initialize
time.sleep(.1)
for device in devices:
utils.run_command(
f'adb -s {device} shell rm -r /storage/sdcard0/Download/*')
utils.log_info(
f'Removed Downloads on {DEVICE_SERIAL_NUMBERS[device]}, ({device})'
)
def start_profiling(self, target_args):
"""Start simpleperf reocrd process on device."""
args = ['/data/local/tmp/simpleperf', 'record', '-o', '/data/local/tmp/perf.data',
self.args.record_options]
if self.adb.run(['shell', 'ls', NATIVE_LIBS_DIR_ON_DEVICE, '>/dev/null', '2>&1']):
args += ['--symfs', NATIVE_LIBS_DIR_ON_DEVICE]
args += ['--log', self.args.log]
args += target_args
adb_args = [self.adb.adb_path, 'shell'] + args
log_info('run adb cmd: %s' % adb_args)
self.record_subproc = subprocess.Popen(adb_args)
def create_genesis_block(self):
genesis_block = Block(previous_block_hash=0,
transactions=[],
height=0,
mining_difficulty=1,
start_nounce=self.chain_id *
self.nounce_distance)
genesis_block = genesis_block.mine()
self.add_block(genesis_block)
log_info("Genesis Block Hash: {}".format(genesis_block.block_hash))
return
def get_user(self):
"""
Attempt to retrieve the user information from the receipt.
"""
try:
return self.decoded['user']['value']
except KeyError:
# If somehow we got a valid receipt without a uuid
# that's a problem. Log here.
log_info('No user in receipt')
raise InvalidReceipt('NO_USER')
def run():
def done(_):
log_info('Job done at {}'.format(datetime.now().strftime("%Y%m%d %H:%M:%S")))
log_info('Job start at {}'.format(datetime.now().strftime("%Y%m%d %H:%M:%S")))
d = get_marketId()
d.addCallback(get_urls)
d.addCallback(get_prices)
d.addBoth(done)
reactor.callLater(interval, run)
def get_user(self):
"""
Attempt to retrieve the user information from the receipt.
"""
try:
return self.decoded['user']['value']
except KeyError:
# If somehow we got a valid receipt without a uuid
# that's a problem. Log here.
log_info('No user in receipt')
raise InvalidReceipt('NO_USER')
def get_inapp_id(self):
"""
Attempt to retrieve the inapp id
from the storedata in the receipt.
"""
try:
return int(self.get_storedata()['inapp_id'])
except Exception, e:
# There was some value for storedata but it was invalid.
log_info('Invalid store data for inapp id: {err}'.format(
err=e))
raise InvalidReceipt('WRONG_STOREDATA')
def _copy_to_binary_cache(self, from_path, expected_build_id, target_file):
if target_file[0] == '/':
target_file = target_file[1:]
target_file = target_file.replace('/', os.sep)
target_file = os.path.join(self.binary_cache_dir, target_file)
if not self._need_to_copy(from_path, target_file, expected_build_id):
# The existing file in binary_cache can provide more information, so no need to copy.
return
target_dir = os.path.dirname(target_file)
if not os.path.isdir(target_dir):
os.makedirs(target_dir)
log_info('copy to binary_cache: %s to %s' % (from_path, target_file))
shutil.copy(from_path, target_file)
def check_purchase(self):
"""
Verifies that the app has been purchased.
"""
sql = """SELECT id, type FROM addon_purchase
WHERE addon_id = %(addon_id)s
AND uuid = %(uuid)s LIMIT 1;"""
self.cursor.execute(sql, {'addon_id': self.addon_id,
'uuid': self.uuid})
result = self.cursor.fetchone()
if not result:
log_info('Invalid receipt, no purchase')
raise InvalidReceipt('NO_PURCHASE')
if result[-1] in (CONTRIB_REFUND, CONTRIB_CHARGEBACK):
log_info('Valid receipt, but refunded')
raise RefundedReceipt
elif result[-1] in (CONTRIB_PURCHASE, CONTRIB_NO_CHARGE):
log_info('Valid receipt')
return
else:
log_info('Valid receipt, but invalid contribution')
raise InvalidReceipt('WRONG_PURCHASE')
def check_url(self):
"""
Verifies that the URL of the verification is what we expect.
"""
path = self.environ['PATH_INFO']
parsed = urlparse(self.decoded.get('verify', ''))
if parsed.netloc not in settings.DOMAIN:
log_info('Receipt had invalid domain')
raise InvalidReceipt
if parsed.path != path:
log_info('Receipt had the wrong path')
raise InvalidReceipt
def check_purchase_app(self):
"""
Verifies that the app has been purchased by the user.
"""
self.setup_db()
sql = """SELECT type FROM addon_purchase
WHERE addon_id = %(app_id)s
AND uuid = %(uuid)s LIMIT 1;"""
self.cursor.execute(sql, {'app_id': self.get_app_id(),
'uuid': self.get_user()})
result = self.cursor.fetchone()
if not result:
log_info('Invalid receipt, no purchase')
raise InvalidReceipt('NO_PURCHASE')
self.check_purchase_type(result[0])
def profile(self):
log_info('prepare profiling')
self.prepare()
log_info('start profiling')
self.start()
self.wait_profiling()
log_info('collect profiling data')
self.collect_profiling_data()
log_info('profiling is finished.')
def _annotate_file(self, from_path, to_path, file_period, is_java):
"""Annotate a source file.
Annotate a source file in three steps:
1. In the first line, show periods of this file.
2. For each function, show periods of this function.
3. For each line not hitting the same line as functions, show
line periods.
"""
log_info('annotate file %s' % from_path)
with open(from_path, 'r') as rf:
lines = rf.readlines()
annotates = {}
for line in file_period.line_dict.keys():
annotates[line] = self._get_percentage_str(file_period.line_dict[line], True)
for func_name in file_period.function_dict.keys():
func_start_line, period = file_period.function_dict[func_name]
if func_start_line == -1:
continue
line = func_start_line - 1 if is_java else func_start_line
annotates[line] = '[func] ' + self._get_percentage_str(period, True)
annotates[1] = '[file] ' + self._get_percentage_str(file_period.period, True)
max_annotate_cols = 0
for key in annotates:
max_annotate_cols = max(max_annotate_cols, len(annotates[key]))
empty_annotate = ' ' * (max_annotate_cols + 6)
dirname = os.path.dirname(to_path)
if not os.path.isdir(dirname):
os.makedirs(dirname)
with open(to_path, 'w') as wf:
for line in range(1, len(lines) + 1):
annotate = annotates.get(line)
if annotate is None:
if not lines[line-1].strip():
annotate = ''
else:
annotate = empty_annotate
else:
annotate = '/* ' + annotate + (
' ' * (max_annotate_cols - len(annotate))) + ' */'
wf.write(annotate)
wf.write(lines[line-1])
def ok_or_expired(self):
# This receipt is ok now let's check it's expiry.
# If it's expired, we'll have to return a new receipt
try:
expire = int(self.decoded.get('exp', 0))
except ValueError:
log_info('Error with expiry in the receipt')
return self.expired()
now = calendar.timegm(gmtime()) + 10 # For any clock skew.
if now > expire:
log_info('This receipt has expired: %s UTC < %s UTC'
% (datetime.utcfromtimestamp(expire),
datetime.utcfromtimestamp(now)))
return self.expired()
return self.ok()
def check_url(self, domain):
"""
Verifies that the URL of the verification is what we expect.
:param domain: the domain you expect the receipt to be verified at,
note that "real" receipts are verified at a different domain
from the main marketplace domain.
"""
path = self.environ['PATH_INFO']
parsed = urlparse(self.decoded.get('verify', ''))
if parsed.netloc != domain:
log_info('Receipt had invalid domain')
raise InvalidReceipt('WRONG_DOMAIN')
if parsed.path != path:
log_info('Receipt had the wrong path')
raise InvalidReceipt('WRONG_PATH')
def check_purchase_inapp(self):
"""
Verifies that the inapp has been purchased.
"""
self.setup_db()
sql = """SELECT inapp_product_id, type FROM stats_contributions
WHERE id = %(contribution_id)s LIMIT 1;"""
self.cursor.execute(
sql,
{'contribution_id': self.get_contribution_id()}
)
result = self.cursor.fetchone()
if not result:
log_info('Invalid receipt, no purchase')
raise InvalidReceipt('NO_PURCHASE')
contribution_inapp_id, purchase_type = result
self.check_purchase_type(purchase_type)
self.check_inapp_product(contribution_inapp_id)
def _check_and_pull_binary(self, binary, expected_build_id, binary_cache_file):
"""If the binary_cache_file exists and has the expected_build_id, there
is no need to pull the binary from device. Otherwise, pull it.
"""
need_pull = True
if os.path.isfile(binary_cache_file):
need_pull = False
if expected_build_id:
build_id = self._read_build_id(binary_cache_file)
if expected_build_id != build_id:
need_pull = True
if need_pull:
target_dir = os.path.dirname(binary_cache_file)
if not os.path.isdir(target_dir):
os.makedirs(target_dir)
if os.path.isfile(binary_cache_file):
os.remove(binary_cache_file)
log_info('pull file to binary_cache: %s to %s' % (binary, binary_cache_file))
self._pull_file_from_device(binary, binary_cache_file)
else:
log_info('use current file in binary_cache: %s' % binary_cache_file)
def decode(self):
"""
Verifies that the receipt can be decoded and that the initial
contents of the receipt are correct.
If its invalid, then just return invalid rather than give out any
information.
"""
try:
receipt = decode_receipt(self.receipt)
except:
log_exception({'receipt': '%s...' % self.receipt[:10],
'addon': self.addon_id})
log_info('Error decoding receipt')
raise InvalidReceipt('ERROR_DECODING')
try:
assert receipt['user']['type'] == 'directed-identifier'
except (AssertionError, KeyError):
log_info('No directed-identifier supplied')
raise InvalidReceipt('NO_DIRECTED_IDENTIFIER')
return receipt
def check_purchase_type(self, purchase_type):
"""
Verifies that the purchase type is of a valid type.
"""
if purchase_type in (CONTRIB_REFUND, CONTRIB_CHARGEBACK):
log_info('Valid receipt, but refunded')
raise RefundedReceipt
elif purchase_type in (CONTRIB_PURCHASE, CONTRIB_NO_CHARGE):
log_info('Valid receipt')
return
else:
log_info('Valid receipt, but invalid contribution')
raise InvalidReceipt('WRONG_PURCHASE')
def check_db(self):
"""
Verifies the decoded receipt against the database.
Requires that decode is run first.
"""
if not self.decoded:
raise ValueError('decode not run')
self.setup_db()
# Get the addon and user information from the installed table.
try:
uuid = self.decoded['user']['value']
except KeyError:
# If somehow we got a valid receipt without a uuid
# that's a problem. Log here.
log_info('No user in receipt')
raise InvalidReceipt('NO_USER')
try:
storedata = self.decoded['product']['storedata']
self.addon_id = int(dict(parse_qsl(storedata)).get('id', ''))
except:
# There was some value for storedata but it was invalid.
log_info('Invalid store data')
raise InvalidReceipt('WRONG_STOREDATA')
sql = """SELECT id, user_id, premium_type FROM users_install
WHERE addon_id = %(addon_id)s
AND uuid = %(uuid)s LIMIT 1;"""
self.cursor.execute(sql, {'addon_id': self.addon_id,
'uuid': uuid})
result = self.cursor.fetchone()
if not result:
# We've got no record of this receipt being created.
log_info('No entry in users_install for uuid: %s' % uuid)
raise InvalidReceipt('WRONG_USER')
pk, self.user_id, self.premium = result
def check_inapp_product(self, contribution_inapp_id):
if int(contribution_inapp_id) != self.get_inapp_id():
log_info('Invalid receipt, inapp_id does not match')
raise InvalidReceipt('NO_PURCHASE')
def __init__(self):
"""
Headers:
========
Header-Name Type Required Description
--------------------------------------------------------
Referer String N Referential url. If seed-file given this is not required.
X-Progress String N The rate of progress of target download file.
X-Client-ID String Y Thunder client id
X-File-Name String given url -> Y,
given seed-file -> N The file name of target download file (UTF-8 encoded). If seed-file given this is not required.
X-File-Size String given url -> Y,
given seed-file -> N The size of target download file. If seed-file given this is not required.
X-Mime-Type String N The mime-type of target download file. If seed-file given this is not required.
X-URL String Y url or seed-file-path
GetParams:params
==========
Parameter Type Required Description
--------------------------------------------------------
key String Y apikey given by mysite.cn must keep it secret.
hash String N file_private_id (hash value generate by thunder client)
digest String Y The digest of url or seed-file
digest-algorithm String Y Digest algorithm
Steps:
======
* Parse query_string
* Auth apikey
* Parse HTTP headers
* Check parameters
* Parse MIME type
"""
self.uuid = uuid.uuid1().hex
env = web.ctx.env
self.error_code = None
self.error_msg = ''
self.error_data = []
params = web.input()
self.key = params.get('key', None)
self.hash = params.get('hash', None)
self.digest = params.get('digest', None)
self.algorithm = params.get('digest-algorithm', None)
if not self.authenticated:
log_normal(logger, {
'action': 'request-unauthorized',
'info': {
'message': 'unauthorized....',
'apikey': self.key
}
}, LOG_INFO, uuid=self.uuid)
raise Unauthorized(wrap_error(ERROR_APIKEY, 'APIKEY Error', ['key']))
self.referer = env.get(HEADER_REFERER, None)
self.last_query = env.get(HEADER_LAST_QUERY, None)
self.cur_query = env.get(HEADER_CUR_QUERY, None)
self.progress = env.get(HEADER_PROGRESS, None)
self.client_addr = env.get(HEADER_CLIENT_ADDR, None)
self.client_id = env.get(HEADER_CLIENT_ID, None)
self.file_name = env.get(HEADER_FILE_NAME, None)
self.file_size = env.get(HEADER_FILE_SIZE, None)
self.mime_type = env.get(HEADER_MIME_TYPE, None)
self.url = env.get(HEADER_URL, None)
self.user_agent = env.get(HEADER_USER_AGENT,None)
self.request_time = None
# PRD Section 3.4 requirement.
try:
log_info(logger_info, {
'action': 'show-request-env',
'info': {
# From: Query String
'key' : self.key,
'hash' : self.hash,
'digest' : self.digest,
'algorithm' : self.algorithm,
# From: Headers
'referer' : self.referer,
'client_id' : self.client_id,
'client_addr' : self.client_addr,
'progress' : self.progress,
'file_name' : self.file_name,
'file_size' : self.file_size,
'mime_type' : self.mime_type,
'url' : self.url,
'user_agent' : self.user_agent,
'request_time' : datetime.now().strftime('%Y-%m-%d %H:%M:%S')
}
}, uuid=self.uuid)
self.request_time = datetime.now()
except UnicodeDecodeError:
self.error_code = ERROR_PARAMS
self.error_msg = 'Parameter encoding unexcept!'
self.error_data.extend(['Header:Referer',
'Header:X-File-Name',
'Header:X-URL'])
message = wrap_error(self.error_code, self.error_msg, self.error_data)
log_normal(logger, {
'action': 'bad-request',
'info': message
}, LOG_WARN, uuid=self.uuid)
raise web.BadRequest(message)
try:
for k in ('last_query', 'cur_query'):
field = getattr(self, k)
if field:
setattr(self, k, datetime(*rfc822.parsedate(field)[:-3]))
except TypeError:
self.error_code = ERROR_PARAMS
self.error_msg = 'If-Modified-Since or Date error!'
self.error_data.append('Header:If-Modified-Since')
self.error_data.append('Header:Date')
if self.file_size:
try:
int(self.file_size)
except ValueError:
self.error_code = ERROR_PARAMS
self.error_msg = 'X-File-Size is not integer!'
self.error_data.append('X-File-Size')
if (not self.client_id
or not self.digest
or not self.algorithm):
# Get parameter or header missing
self.error_code = ERROR_PARAMS
self.error_msg = 'Parameter not enough!'
for filed, tag in [
(self.client_id , 'X-Client-ID'),
(self.digest , 'digest'),
(self.algorithm , 'digest-algorithm')]:
if not filed:
self.error_data.append(tag)
self.is_seed = not self.url or self.url.strip().startswith("file://")
self.ext = None
if self.file_name:
parts = self.file_name.rsplit('.', 1)
if len(parts) >= 2:
self.ext = parts[-1]
self.scheme = url_scheme(self.url)
# If config matched transform the digest(hashinfo) to lower case
if (DIGEST_TRANSFORM
and self.algorithm not in IGNORE_ALGORITHMS
and self.digest):
self.digest = self.digest.lower()
def main():
sys.setrecursionlimit(MAX_CALLSTACK_LENGTH * 2 + 50)
parser = argparse.ArgumentParser(description='report profiling data')
parser.add_argument('-i', '--record_file', nargs='+', default=['perf.data'], help="""
Set profiling data file to report. Default is perf.data.""")
parser.add_argument('-o', '--report_path', default='report.html', help="""
Set output html file. Default is report.html.""")
parser.add_argument('--min_func_percent', default=0.01, type=float, help="""
Set min percentage of functions shown in the report.
For example, when set to 0.01, only functions taking >= 0.01%% of total
event count are collected in the report. Default is 0.01.""")
parser.add_argument('--min_callchain_percent', default=0.01, type=float, help="""
Set min percentage of callchains shown in the report.
It is used to limit nodes shown in the function flamegraph. For example,
when set to 0.01, only callchains taking >= 0.01%% of the event count of
the starting function are collected in the report. Default is 0.01.""")
parser.add_argument('--add_source_code', action='store_true', help='Add source code.')
parser.add_argument('--source_dirs', nargs='+', help='Source code directories.')
parser.add_argument('--add_disassembly', action='store_true', help='Add disassembled code.')
parser.add_argument('--binary_filter', nargs='+', help="""Annotate source code and disassembly
only for selected binaries.""")
parser.add_argument('--ndk_path', nargs=1, help='Find tools in the ndk path.')
parser.add_argument('--no_browser', action='store_true', help="Don't open report in browser.")
parser.add_argument('--show_art_frames', action='store_true',
help='Show frames of internal methods in the ART Java interpreter.')
args = parser.parse_args()
# 1. Process args.
binary_cache_path = 'binary_cache'
if not os.path.isdir(binary_cache_path):
if args.add_source_code or args.add_disassembly:
log_exit("""binary_cache/ doesn't exist. Can't add source code or disassembled code
without collected binaries. Please run binary_cache_builder.py to
collect binaries for current profiling data, or run app_profiler.py
without -nb option.""")
binary_cache_path = None
if args.add_source_code and not args.source_dirs:
log_exit('--source_dirs is needed to add source code.')
build_addr_hit_map = args.add_source_code or args.add_disassembly
ndk_path = None if not args.ndk_path else args.ndk_path[0]
# 2. Produce record data.
record_data = RecordData(binary_cache_path, ndk_path, build_addr_hit_map)
for record_file in args.record_file:
record_data.load_record_file(record_file, args.show_art_frames)
record_data.limit_percents(args.min_func_percent, args.min_callchain_percent)
def filter_lib(lib_name):
if not args.binary_filter:
return True
for binary in args.binary_filter:
if binary in lib_name:
return True
return False
if args.add_source_code:
record_data.add_source_code(args.source_dirs, filter_lib)
if args.add_disassembly:
record_data.add_disassembly(filter_lib)
# 3. Generate report html.
report_generator = ReportGenerator(args.report_path)
report_generator.write_script()
report_generator.write_content_div()
report_generator.write_record_data(record_data.gen_record_info())
report_generator.finish()
if not args.no_browser:
open_report_in_browser(args.report_path)
log_info("Report generated at '%s'." % args.report_path)
