def reducer(acc, store):
if 'street_crossing_name' in store.values:
acc['street_crossing_name'] = _.get(acc, 'street_crossing_name', 0) + 1
elif 'first_name' in store.values:
acc['first_name'] = _.get(acc, 'first_name', 0) + 1
elif 'street_address' in store.values:
acc['street_address'] = _.get(acc, 'street_address', 0) + 1
return acc
def create_modified_json(reference_files):
config = _get_json_data(os.path.join(BASE_DIR, 'config.json'))
refer_link = _.get(config, 'refer_link')
origin_json_files = _get_json_file_path(TARGET_DIR)
if len(origin_json_files) == 0:
_error(f"JSON not found at ({TARGET_DIR})")
else:
total_dictionary = {}
for json_path in origin_json_files:
fname = get_file_syntax(json_path)
full_single_json = _get_json_data(json_path)
single = _.get(full_single_json, 'files')
total_dictionary[key_path_creator(json_path)] = full_single_json
for obj in single:
new_desc = _modify_yaml_to_json(obj['description'])
_.set(obj, 'description', new_desc)
package = obj['package']
enums = get_enums_dict(obj['enums'])
services = _.get(obj, 'services')
messages = _.get(obj, 'messages')
message_names = [a['name'] for a in messages]
for service in services:
for method in service['methods']:
method_status = is_exists_in_same_file(message_names, method, fname)
svc_desc = _modify_yaml_to_json(method['description'])
method['description'] = svc_desc
google_api = _.get(method, 'options')
if(google_api is not None):
rest_apis = google_api['google.api.http']['rules']
method['restAPI'] = rest_apis
if (method['requestFullType'] in [*refer_link] and not method_status['req_status']):
method['requestFullType_link'] = refer_link[method['requestFullType']]
if (method['responseFullType'] in [*refer_link] and not method_status['res_status']):
method['responseFullType_link'] = refer_link[method['responseFullType']]
for message in messages:
for field in message["fields"]:
field_long_type_wrapper(message, field, enums)
field['description'] = field_desc_wrapper(field['description'])
field_full_type_wrapper(fname, field, refer_link, package)
full_single_json['file_info'] = fname
return total_dictionary
def fetch_release_notes_from_prs(
github_helper: GitHubRepositoryHelper,
pr_numbers_in_range: typing.Set[str],
cn_current_repo: ComponentName) -> [ReleaseNote]:
# we should consider adding a release-note label to the PRs
# to reduce the number of search results
prs_iter = github_helper.search_issues_in_repo('type:pull is:closed')
release_notes = list()
for pr_iter in prs_iter:
pr_dict = pr_iter.as_dict()
pr_number = str(pr_dict['number'])
if pr_number not in pr_numbers_in_range:
continue
release_notes_pr = extract_release_notes(
reference_id=pr_number,
text=pr_dict['body'],
user_login=_.get(pr_dict, 'user.login'),
cn_current_repo=cn_current_repo,
reference_type=REF_TYPE_PULL_REQUEST)
if not release_notes_pr:
continue
release_notes.extend(release_notes_pr)
return release_notes
def _get_human_managed_documents_links() -> dict:
"""
Get human managed documents URL link from config.json
Args:
Returns:
dict
"""
config = _get_json_data(os.path.join(BASE_DIR, 'config.json'))
human_doc_links = _.get(config, 'human_doc_links')
return human_doc_links
if directory == 'googleEs':
find['googleEs'] = args.name
continue
# handle gcid condition
if directory == 'gcid':
find['gcid'] = "gcid:{}".format(args.gcid)
continue
# handle gcid condition
if directory == 'gmb':
find['gmb'] = args.gcid
continue
data = getDirectory(directory)
cats = _.get(data, 'categories', [])
# select the right key to look for
key = "title"
if directory == "apple":
key = "name"
# display selection
print("")
print(directory.upper())
if _.get(find, directory, None) is not None:
print("current category: {}".format(_.get(find, directory,
'None')))
# set the initial search criteria
selection = args.search
def _generate_summary_mds(context_input, managed_link, history):
table_of_contents = {}
title_index = []
config = _get_json_data(os.path.join(BASE_DIR, 'config.json'))
for path in context_input:
file_info = _get_file_name_syntax(path)
full_path = path[path.find('api/') + 4:len(path)].replace(
'.json', '.md')
parsed_path = full_path.split('/')
parsed_path[len(parsed_path) - 1] = parsed_path[len(parsed_path) -
1].replace('_', '-')
full_file_name = '/'.join(parsed_path)
parsed_path[len(parsed_path) - 1] = file_info['title']
base_bullet = '* '
check_key = ""
for i in range(len(parsed_path)):
if i != 0:
check_key = check_key + '.' + parsed_path[i]
base_bullet = ' ' + base_bullet
else:
check_key = parsed_path[i]
title_no_under_bar = parsed_path[i].replace('_', ' ')
updated_header = base_bullet + '[' + title_no_under_bar + ']'
if _.get(table_of_contents, check_key, None) is None:
if i == (len(parsed_path) - 1):
_.set(table_of_contents, check_key, {})
title_index.append({
'title': updated_header.title(),
'url': full_file_name.lower()
})
else:
mk_path = check_key.replace('.', '/')
readme_url = mk_path + '/' + 'README.md'
readme_path = os.path.join(BASE_DIR, mk_path, 'README.md')
if not os.path.exists(readme_path):
_generate_md_file(readme_path, TEMPLATE_NAMES[2], '')
_.set(table_of_contents, check_key, {})
title_index.append({
'title': updated_header.title(),
'url': readme_url
})
output_to_create = os.path.join(BASE_DIR, 'SUMMARY.md')
normalized_link = _normalize_managed_link(managed_link)
context_input = {
'toc': title_index,
'managed_link': normalized_link,
'history': {
'cur_version': history
}
}
# Updating All SUMMARY.md
_generate_md_file(output_to_create, TEMPLATE_NAMES[1], context_input)
# Updating All README.md
output_to_create = os.path.join(BASE_DIR, 'README.md')
readme_output = _.get(config, 'intro_comment')
version_info = history if history.startswith('v') else 'v' + history
_.set(readme_output, 'version', version_info)
_generate_md_file(output_to_create, TEMPLATE_NAMES[2], readme_output)
def _create_modified_json() -> dict:
"""
Generate JSON2 that has mapping data according to Jinja templates
Args:
Returns:
dict
"""
config = _get_json_data(os.path.join(BASE_DIR, 'config.json'))
refer_link = _.get(config, 'refer_link')
origin_json_files = _get_json_file_path(TARGET_DIR)
if len(origin_json_files) == 0:
_error(f"JSON not found at ({TARGET_DIR})")
else:
total_dictionary = {}
for json_path in origin_json_files:
# Get file all file paths such as full path, file name, file name in lower case, etc
file_name_syntax = _get_file_name_syntax(json_path)
# Get full JSON data by its file path
full_single_json = _get_json_data(json_path)
# Get only proto buffer data from JSON file, No scalarValueTypes
single = full_single_json.get('files')
# Set full data json by its path and JSON file name
total_dictionary[_key_path_creator(json_path)] = full_single_json
# iterate over single dictionary file data
for obj in single:
obj['description'] = _modify_yaml_to_json(
obj.get('description'))
package = obj.get('package')
enums = _get_enums_dict(obj.get('enums'))
services = obj.get('services')
messages = obj.get('messages')
message_names = [msg['name'] for msg in messages]
# processing services in JSON data
for service in services:
for method in service.get('methods', []):
# find referred contents in the service to insert link in markdown
method_status = _is_exists_in_same_file(
message_names, method, file_name_syntax)
# categorize and set description into managed key for markdown template
method['description'] = _desc_wrapper(
method['description'])
# get rest API such as post, or get method
google_api = method.get('options')
# parsing all all service data into dict with appropriate key to mapping into Jinja template
if google_api is not None:
rest_apis = google_api.get('google.api.http',
{}).get('rules', [])
method['restAPI'] = rest_apis
if method.get('requestFullType') in [
*refer_link
] and not method_status['req_status']:
method['requestFullType_link'] = refer_link[
method['requestFullType']]
if method.get('responseFullType') in [
*refer_link
] and not method_status['res_status']:
method['responseFullType_link'] = refer_link[
method['responseFullType']]
# processing messages in JSON data
for message in messages:
# Check is_required column may requires in markdown
is_require_exits = False
for idx, field in enumerate(message["fields"]):
# set fields referred to ENUM such as google.protobuf.Struct.
_field_long_type_wrapper(message, field, enums)
# categorize and set description into managed key for markdown template
field['description'] = _desc_wrapper(
field['description'])
_field_full_type_wrapper(file_name_syntax, field,
refer_link, package)
# set key is_required and its value for required column in markdown
if not is_require_exits and 'is_required' in field[
'description']:
is_require_exits = True
if not is_require_exits:
message['no_requires'] = 'true'
full_single_json['file_info'] = file_name_syntax
return total_dictionary
def get_auto_wbnb_price():
pancake_swap_price = requests.get(PANCAKE_PRICE_URL).json()
auto_price_usd = Decimal(_.get(pancake_swap_price, 'prices.AUTO', 0))
wbnb_price_usd = Decimal(_.get(pancake_swap_price, 'prices.WBNB', 0))
return auto_price_usd, wbnb_price_usd
def auto_compund(wallet_address, private_key, min_amount_to_harvest):
w3 = Web3(Web3.HTTPProvider('https://bsc-dataseed1.binance.org:443'))
auto_contract = w3.eth.contract(address=AUTO_CONTRACT, abi=AUTO_ABI)
pancake_swap_contract = w3.eth.contract(address=PANCAKE_SWAP_CONTRACT,
abi=PANCAKE_SWAP_ABI)
auto_farms_data = requests.get(AUTO_FARM_INFO_URL).json()
auto_pool_ids = _.get(auto_farms_data, 'pools', {}).keys()
def get_auto_wbnb_price():
pancake_swap_price = requests.get(PANCAKE_PRICE_URL).json()
auto_price_usd = Decimal(_.get(pancake_swap_price, 'prices.AUTO', 0))
wbnb_price_usd = Decimal(_.get(pancake_swap_price, 'prices.WBNB', 0))
return auto_price_usd, wbnb_price_usd
auto_price_usd, wbnb_price_usd = get_auto_wbnb_price()
if not auto_pool_ids or not auto_price_usd or not wbnb_price_usd:
print("Missing pool and/or price data. Exiting...")
exit()
def withdraw_auto_token_if_necessary(acc_auto_withdraw_gwei, pool_id):
pool_id = int(pool_id)
current_pending_auto_gwei = auto_contract.functions.pendingAUTO(
pool_id, wallet_address).call()
current_pending_auto_eth = w3.fromWei(current_pending_auto_gwei,
'ether')
if current_pending_auto_eth * auto_price_usd > min_amount_to_harvest:
print(
f"- Current pending auto for pool id {pool_id}: {current_pending_auto_eth} = ${current_pending_auto_eth * auto_price_usd:.2f}. Withdrawing...",
end='')
# withdrawing a pool with 0 only withdraw the reward = Harvest
tx = auto_contract.functions.withdraw(pool_id, 0).buildTransaction(
{
'from': wallet_address,
'nonce': w3.eth.getTransactionCount(wallet_address),
'gas': 500000
})
signed_tx = w3.eth.account.signTransaction(tx, private_key)
tx_hash = w3.eth.sendRawTransaction(signed_tx.rawTransaction)
tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)
print(
f"...done.\n tx hash = {tx_receipt['transactionHash'].hex()}")
return acc_auto_withdraw_gwei + current_pending_auto_gwei
return acc_auto_withdraw_gwei
connected = w3.isConnected()
if connected:
print(
f"Checking each pool to see if auto rewards meet the ${min_amount_to_harvest} threshold"
)
harvested_auto_amt_gwei = _.reduce(auto_pool_ids,
withdraw_auto_token_if_necessary, 0)
harvested_auto_amt_eth = w3.fromWei(harvested_auto_amt_gwei, 'ether')
print(
f"- Total harvested: Auto {harvested_auto_amt_eth} = ${harvested_auto_amt_eth * auto_price_usd:.2f}"
)
if harvested_auto_amt_gwei > 0:
## SELL HALF AUTO FOR WBNB
half_harvested_auto_amt_gwei = int(harvested_auto_amt_gwei / 2)
# get expceted WBNB amount from pancake
amounts_out = pancake_swap_contract.functions.getAmountsOut(
amountIn=half_harvested_auto_amt_gwei,
path=[AUTO_TOKEN_CONTRACT, WRAP_BNB_TOKEN_CONTRACT],
).call()
wbnb_amt_gwei = int(amounts_out[1] *
(1 - PANCAKE_ASSUMED_SLIPPAGE))
print(
f"- Swapping AUTO {half_harvested_auto_amt_gwei} for WBNB {wbnb_amt_gwei} on pancakeswap...",
end='')
tx = pancake_swap_contract.functions.swapExactTokensForTokens(
amountIn=half_harvested_auto_amt_gwei,
amountOutMin=wbnb_amt_gwei,
path=[AUTO_TOKEN_CONTRACT, WRAP_BNB_TOKEN_CONTRACT],
to=wallet_address,
deadline=int(time()) + 60 * 5 # give 5 minutes deadline
).buildTransaction({
'from':
wallet_address,
'nonce':
w3.eth.getTransactionCount(wallet_address),
'gas':
500000
})
signed_tx = w3.eth.account.signTransaction(tx, private_key)
tx_hash = w3.eth.sendRawTransaction(signed_tx.rawTransaction)
tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)
print(
f"...done.\n tx hash = {tx_receipt['transactionHash'].hex()}")
transfer_log = _.find(
tx_receipt['logs'],
lambda x: x['address'] == WRAP_BNB_TOKEN_CONTRACT)
if not transfer_log or not transfer_log.get('data'):
print(
f"Error: Could not find log for the WBNB transfer. Transaction must have failed."
)
exit(-1)
## ADD LIQUIDITY TO PANCAKE SWAP. SWAP AUTO & WBNB FOR WBNB-AUTO LP TOKEN
received_wbnb_amt_gwei = int(transfer_log['data'], 16)
wbnb_amt_gwei = int(received_wbnb_amt_gwei *
(1 - PANCAKE_ASSUMED_SLIPPAGE))
print(
f"- Add lidquidity AUTO-WBNB: AUTO {half_harvested_auto_amt_gwei} for WBNB {wbnb_amt_gwei} on pancakeswap...",
end='')
tx = pancake_swap_contract.functions.addLiquidity(
tokenA=AUTO_TOKEN_CONTRACT,
tokenB=WRAP_BNB_TOKEN_CONTRACT,
amountADesired=half_harvested_auto_amt_gwei,
amountBDesired=wbnb_amt_gwei,
amountAMin=0,
amountBMin=0,
to=wallet_address,
deadline=int(time()) + 60 * 5 # give 5 minutes deadline
).buildTransaction({
'from':
wallet_address,
'nonce':
w3.eth.getTransactionCount(wallet_address),
'gas':
500000
})
signed_tx = w3.eth.account.signTransaction(tx, private_key)
tx_hash = w3.eth.sendRawTransaction(signed_tx.rawTransaction)
tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)
print(
f"...done.\n tx hash = {tx_receipt['transactionHash'].hex()}")
def find_transfer_lp_logs(log):
# Search for the log transfering money to your wallet
# Transfer log have 3 topics: function hash, from, to.
# Looking for `from = 0x0` & `to = wallet_address`
topics = log['topics']
if len(topics) == 3 and topics[1].hex() == ROOT_ADDR:
# extract the leading 0 from the address in the log
addr = '0x' + topics[2].hex()[ADDR_SIZE -
len(wallet_address) + 2:]
return addr == wallet_address.lower()
transfer_lp_log = _.find(tx_receipt['logs'], find_transfer_lp_logs)
if not transfer_lp_log or not transfer_lp_log.get('data'):
print(
f"Error: Could not find log for the LP transfer. Transaction must have failed."
)
exit(-1)
liquidity_created = int(transfer_lp_log['data'], 16)
print(
f"- Add {w3.fromWei(liquidity_created, 'ether')} token back into AUTO-WBNB LP vault...",
end='')
tx = auto_contract.functions.deposit(
AUTO_WBNB_POOL_ID, liquidity_created).buildTransaction({
'from':
wallet_address,
'nonce':
w3.eth.getTransactionCount(wallet_address),
'gas':
500000
})
signed_tx = w3.eth.account.signTransaction(tx, private_key)
tx_hash = w3.eth.sendRawTransaction(signed_tx.rawTransaction)
tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)
print(f"...end.\n tx hash = {tx_receipt['transactionHash'].hex()}")
else:
print("Connection Error!")
role_config_group_resource_api = cm_client.RoleConfigGroupsResourceApi(
api_client)
config_by_roles = role_config_group_resource_api.read_role_config_groups(
cluster_name=cluster, service_name=service)
config_by_group = {}
role_config_group_name_displayname_mapping = {}
for item in config_by_roles.items:
role_group_name = item.name
_.set(role_config_group_name_displayname_mapping, "" + item.display_name,
role_group_name)
for subitem in item.config.items:
_.set(
config_by_group, "" + subitem.name,
_.push(_.get(config_by_group, "" + subitem.name, []),
role_group_name))
updating_counter = 0
updated_counter = 0
for config in args.config:
picked_role_config_group = None
configuration = _.replace(config[0], '.', '_')
value = config[1]
specified_role_config_group = _.get(config, '[2]', None)
specified_role_config_group_display_name = _.get(config, '[2]', None)
if (specified_role_config_group_display_name
and _.get(role_config_group_name_displayname_mapping,
"" + specified_role_config_group_display_name, None)):
specified_role_config_group = role_config_group_name_displayname_mapping[
specified_role_config_group_display_name]
query = "*"
if args.since:
query = "date%%3A>now-%s" % args.since
if args.query:
query = "%s AND (%s)" % (query, args.query)
if args.action == "showbrands":
r = session.get("https://pro.urlscan.com/api/v1/pro/kits")
if not r.status_code == requests.codes.ok:
logging.error("Error fetching brand definitions: %s" % r.json())
sys.exit(1)
for kit in r.json()["kits"]:
print("="*80)
print("%s - %s (%s)\nKey: %s\nWhitelisted domains: %s" % (
kit["name"],
_.head(_.get(kit, "vertical", [])),
_.head(_.get(kit, "country", [])),
kit["key"],
_.get(kit, "terms.domains", [])
))
print("URL: https://pro.urlscan.com/search?filter=%%24phishing_%s" % kit["key"])
print("API: https://pro.urlscan.com/api/v1/pro/search?filter=%%24phishing_%s" % kit["key"])
elif args.action == "showlatest":
r = session.get("https://pro.urlscan.com/api/v1/pro/search?q=%s&filter=$phishing_%s&size=%d" % (query, args.brand, args.limit))
print("\nSearching for brand '%s' with query '%s' and limit '%d'" % (args.brand, query, args.limit))
print("Show in Pro: https://pro.urlscan.com/search?query=%s&filter=$phishing_%s" % (query, args.brand))
if not r.status_code == requests.codes.ok:
logging.error("Error fetching brand definitions: %s" % r.json())
sys.exit(1)
print("%d/%d results returned\n\n" % (len(r.json()["results"]), r.json()["total"]))
for res in r.json()["results"]:
def get_or_call(obj, path):
value = _.get(obj, path)
if callable(value):
return value()
return value
def _get_human_doc_links():
config = _get_json_data(os.path.join(BASE_DIR, 'config.json'))
human_doc_links = _.get(config, 'human_doc_links')
return human_doc_links
def _generate_summary_mds(context_input, managed_link, history):
table_of_contents = {}
title_index = []
config = _get_json_data(os.path.join(BASE_DIR, 'config.json'))
for path in context_input:
file_info = get_file_syntax(path)
full_path = path[path.find('api/') + 4:len(path)].replace('.json', '.md')
parsed_path = full_path.split('/')
parsed_path[len(parsed_path) - 1] = parsed_path[len(parsed_path) - 1].replace('_', '-')
full_file_name = '/'.join(parsed_path)
parsed_path[len(parsed_path)-1] = file_info['title']
base_bullet = '* '
check_key = ""
for i in range(len(parsed_path)):
if(i != 0 ):
check_key = check_key+'.'+parsed_path[i]
base_bullet = ' ' + base_bullet
else:
check_key = parsed_path[i]
updated_header = base_bullet + '[' + parsed_path[i] + ']'
if(_.get(table_of_contents, check_key, None) == None ):
if(i == (len(parsed_path)-1)):
_.set(table_of_contents, check_key, {})
title_index.append({'title': updated_header.title(), 'url': full_file_name})
else:
mk_path = check_key.replace('.', '/')
readme_url = mk_path + '/'+ 'README.md'
readme_path = os.path.join(BASE_DIR, mk_path, 'README.md')
if not os.path.exists(readme_path):
_generate_md_file(readme_path, TEMPLATE_NAMES[2], '')
_.set(table_of_contents, check_key, {})
title_index.append({'title': updated_header.title(), 'url': readme_url})
if ('version_record' in history and len(history['version_record']) > 0):
vtable_of_contents = []
gitbook_space = _.get(config, 'refer_link.git_book_space')
previous_version_md = os.path.join(BASE_DIR, 'previous_version', 'README.md')
for version in history['version_record']:
line = f'* [{version}]({gitbook_space}previous-versions/previous_version/{version}/)'
vtable_of_contents.append(line)
_generate_md_file(previous_version_md, TEMPLATE_NAMES[3], {'list': vtable_of_contents})
output_to_create = os.path.join(BASE_DIR, 'SUMMARY.md')
context_input = {'toc': title_index}
context_input['managed_link'] = _normalize_managed_link(managed_link)
context_input['history'] = history
# Updating All SUMMARY.md
_generate_md_file(output_to_create, TEMPLATE_NAMES[1], context_input)
# Updating All README.md
output_to_create = os.path.join(BASE_DIR, 'README.md')
README_OUTPUT = _.get(config, 'intro_comment')
_.set(README_OUTPUT, 'version', history['cur_version'])
_generate_md_file(output_to_create, TEMPLATE_NAMES[2], README_OUTPUT)
cm_client.configuration.username = cm_client_username
cm_client.configuration.password = cm_client_password
url = f'{api_protocol}://{api_host}:{port}/api/{api_version}'
api_client = cm_client.ApiClient(url)
role_config_group_resource_api = cm_client.RoleConfigGroupsResourceApi(api_client)
config_by_roles = role_config_group_resource_api.read_role_config_groups(cluster_name=cluster, service_name=service)
config_by_group = {}
for item in config_by_roles.items:
role_group_name = item.name
for subitem in item.config.items:
_.set(config_by_group, f'{subitem.name}', _.push(_.get(config_by_group, f'{subitem.name}', []), role_group_name))
for config in args.config:
picked_role_config_group = None
configuration = _.replace(config[0], '.', '_')
value = config[1]
specified_role_config_group = _.get(config, '[2]', None)
if configuration in config_by_group:
structures_for_config = config_by_group[configuration]
picked_role_config_group = specified_role_config_group or _.find(structures_for_config, lambda x: _.ends_with(x, '-BASE'))
if _.index_of(structures_for_config, picked_role_config_group) != -1:
# updating configuration
print(f"Updating: '{configuration}' for '{picked_role_config_group}' config group")