def line_integration(rmid):
print("Begin process for " + str(rmid))
mjd_list = map(int, os.listdir(Location.project_loca + "data/raw/" +
str(rmid)))
os.chdir(Location.project_loca + "/result/flux_of_line/")
try:
os.mkdir(str(rmid))
except OSError:
pass
pool = Pool(processes=32)
m = Manager()
lock = m.Lock()
fe2dic = m.dict()
hbetandic = m.dict()
hbetabdic = m.dict()
o3dic = m.dict()
contdic = m.dict()
func = partial(line_integration_single, rmid, lock, fe2dic, hbetandic,
hbetabdic, o3dic, contdic)
pool.map(func, mjd_list)
output_flux(rmid, dict(fe2dic), "Fe2")
output_flux(rmid, dict(hbetandic), "Hbetan")
output_flux(rmid, dict(hbetabdic), "Hbetab")
output_flux(rmid, dict(contdic), "cont")
output_flux(rmid, dict(o3dic), "O3")
pool.close()
pool.join()
def build_av_tf_idf_dv(docs, doc_num, model, save=True, save_file="doc_vector_tfidf.bin"):
docs = list(docs)
vectorizer = CountVectorizer()
tfidf_transformer = TfidfTransformer()
count_fv = vectorizer.fit_transform(util.word2sentence(docs))
tfidf_fv = tfidf_transformer.fit_transform(count_fv)
num_features = model.syn0.shape[1]
manager = Manager()
global_word_set = manager.dict(util.get_word_vec_dict(model))
global_vocabulary = manager.dict(vectorizer.vocabulary_);
global_doc_vector = mp.Array('d', doc_num*num_features, lock=False)
pool = mp.Pool(initializer=initprocess, initargs=[global_doc_vector])
index = 0
# test(docs[0], global_word_set, 0, doc_num, global_vocabulary, global_doc_vector, global_tfidf_fv)
for words in docs:
pool.apply_async(single_av_tf_idf_dv, [words, global_word_set, index, doc_num, global_vocabulary, tfidf_fv[index]])
index += 1
pool.close()
pool.join()
doc_vector = np.frombuffer(global_doc_vector).reshape((doc_num, num_features))
if save:
np.save(save_file, doc_vector)
return doc_vector
def run():
# build the mdp
start = time.time()
room_size = 3
num_rooms = 5
mdp = maze_mdp.MazeMDP(room_size=room_size, num_rooms=num_rooms)
# build the agent
m = Manager()
init_dict = {(s, a): 0 for s in mdp.states for a in mdp.ACTIONS + [None]}
shared_weights = m.dict(init_dict)
shared_value_weights = m.dict(init_dict)
agent = async_actor_critic.AsyncActorCritic(actions=mdp.ACTIONS, discount=mdp.DISCOUNT,
weights=shared_weights, value_weights=shared_value_weights, tau=.3, learning_rate=.5)
# build a single experiment
rewards = m.list()
start_state_values = m.list()
max_steps = (2 * room_size * num_rooms) ** 2
exp = experiment.Experiment(mdp=mdp, agent=agent, num_episodes=800, max_steps=max_steps,
rewards=rewards, start_state_values=start_state_values)
# run the experiment
multiexperiment = experiment.MultiProcessExperiment(experiment=exp, num_agents=NUM_PROCESSES)
multiexperiment.run()
# report results
end = time.time()
print 'took {} seconds to converge'.format(end - start)
mdp.print_state_values(shared_value_weights)
optimal = mdp.EXIT_REWARD + (2 * room_size * num_rooms * mdp.MOVE_REWARD)
utils.plot_values(rewards, optimal, 'rewards')
utils.plot_values(start_state_values, optimal, 'start state value')
def process_job_parallel(scheduler, job, nr_cores, nr_iter, parameters = None):
Logger.log_level = 2
processes = []
manager = Manager()
return_values = manager.dict()
extremes = manager.dict()
start_time = datetime.datetime.now()
for i in range(nr_cores):
p = Process(target=worker, args=(i, nr_cores, scheduler, job, nr_iter, return_values, extremes, parameters,))
processes.append(p)
p.start()
for process in processes:
process.join()
#reduce
results = []
for value in return_values.values():
for entry in value:
results.append(entry)
min = None
max = None
for extreme in extremes.values():
if min is None or extreme[0].total_time < min.total_time:
min = extreme[0]
if max is None or extreme[1].total_time > max.total_time:
max = extreme[1]
Logger.warning("Min: %s" % min.total_time)
Logger.warning("Max: %s" % max.total_time)
duration = datetime.datetime.now() - start_time
Logger.warning("Simulation complete. Duration: %s" % (duration))
return results, (min,max)
def __init__(self, config):
'''*config* can be obtained from the function :func:`cloudfusion.store.sugarsync.sugarsync_store.SugarsyncStore.get_config`,
but you need to add user and password::
config = SugarsyncStore.get_config()
config['user'] = '******' #your account username/e-mail address
config['password'] = '******' #your account password
Or you can use a configuration file that already has password and username set by specifying a path::
path_to_my_config_file = '/home/joe/MySugarsync.ini'
config = get_config(path_to_my_config_file)
:param config: dictionary with key value pairs'''
#self.dir_listing_cache = {}
self._logging_handler = 'sugarsync'
self.logger = logging.getLogger(self._logging_handler)
self.logger = db_logging_thread.make_logger_multiprocessingsave(self.logger)
manager = Manager()
self.path_cache = manager.dict()
# use a lock for synchronized appends
self._dir_listing_cache = manager.dict()
self._dir_listing_cache_lock = RLock()
self._last_partial_cache = manager.list()
self._time_of_last_partial_cache = 0
#error handling for authorization error
self.root = config["root"]
try:
self.client = SugarsyncClient(config)
except Exception, e:
raise StoreAutorizationError(repr(e), 0)
def main_parallel():
Component.resetPfKeeping()
Component.resetCostKeeping()
manager = Manager()
Component.pfkeeping = manager.dict(Component.pfkeeping)
Component.costkeeping = manager.dict(Component.costkeeping)
pool = Pool(processes=3)
toolbox.register("map", pool.map)
print "MULTIOBJECTIVE OPTIMIZATION: parallel version"
start_delta_time = time.time()
# optimization
random.seed(64)
npop = 100
ngen = 50
stats = tools.Statistics(key=lambda ind: ind.fitness.values)
stats.register("avg", np.mean, axis=0)
stats.register("std", np.std, axis=0)
stats.register("min", np.min, axis=0)
stats.register("max", np.max, axis=0)
logbook = tools.Logbook()
logbook.header = "gen", "evals", "avg", "std", "min", "max"
pop = toolbox.population(n=npop)
fits = toolbox.map(toolbox.evaluate, pop)
for fit,ind in zip(fits, pop):
ind.fitness.values = fit
nevals = npop
allpop = []
for gen in range(ngen):
allpop = allpop+pop
record = stats.compile(pop)
logbook.record(gen=gen, evals=nevals, **record)
print(logbook.stream)
offspring = algorithms.varOr(pop, toolbox, lambda_=npop, cxpb=0.5, mutpb=0.1)
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
nevals = len(invalid_ind)
fits = toolbox.map(toolbox.evaluate, invalid_ind)
for fit,ind in zip(fits, invalid_ind):
ind.fitness.values = fit
pop = toolbox.select(offspring+pop, k=npop)
front = toolbox.sort(allpop, k=int(ngen*npop), first_front_only=True)
pool.close()
pool.join()
delta_time = time.time() - start_delta_time
print 'DONE: {} s'.format(str(datetime.timedelta(seconds=delta_time)))
return allpop, logbook, front
def main():
parser = argparse.ArgumentParser(description='Takes deduplicated bam files and preprocess\'s for analysis\n')
parser.add_argument('-c', '--config', help='Conditions containing Sam/Bam files, values are naming', required=True)
parser.add_argument('-g', '--genome', help='Genome the samples are aligned to, options include mm10/mm9/hg19', required=True)
parser.add_argument('-o', '--outdir', help='Output directory, will create transdense, nfree and npres directories', required=True)
parser.add_argument('-t', '--threads', help='threads, default=1', default=1, required=False)
parser.add_argument('-b', action='store_true', help='Use if Config contains bam files', required=False)
parser.add_argument('-n', action='store_true', help='Runs just nfree <60 and >60', required=False)
if len(sys.argv)==1:
parser.print_help()
sys.exit(1)
args = vars(parser.parse_args())
Config = ConfigParser.ConfigParser()
Config.optionxform = str
Config.read(args["config"])
conditions = ConfigSectionMap("Conditions", Config)
chrom = pkg_resources.resource_filename('pyatactools', 'data/{}.chrom.sizes'.format(args["genome"]))
if not os.path.isfile(chrom):
raise Exception("Unsupported Genome!")
transdense_dir = os.path.join(args["outdir"], "transdense")
nfree_dir = os.path.join(args["outdir"], "nfree")
npres_dir = os.path.join(args["outdir"], "npres")
pool = Pool(int(args["threads"]))
if not os.path.isdir(transdense_dir):
os.makedirs(transdense_dir)
os.makedirs(nfree_dir)
os.makedirs(npres_dir)
ddup_bams = list(conditions.keys())
if args["n"]:
manager = Manager()
return_dict = manager.dict()
pool = Pool(int(args["threads"]))
return_dict = manager.dict()
nfree_dir1 = os.path.join(args["outdir"], "nfree_small")
nfree_dir2 = os.path.join(args["outdir"], "nfree_large")
if not os.path.isdir(nfree_dir1):
os.makedirs(nfree_dir1)
os.makedirs(nfree_dir2)
pool.map(function5, itertools.izip(ddup_bams, itertools.repeat(nfree_dir1),itertools.repeat(nfree_dir2), itertools.repeat(return_dict)))
pool.map(function4, itertools.izip(list(return_dict.keys()), itertools.repeat(chrom)))
else:
manager = Manager()
return_dict = manager.dict()
pool = Pool(int(args["threads"]))
pool.map(function1, itertools.izip(ddup_bams, itertools.repeat(transdense_dir), itertools.repeat(return_dict)))
pool.map(function4, itertools.izip(list(return_dict.keys()), itertools.repeat(chrom)))
return_dict = manager.dict()
pool.map(function2, itertools.izip(ddup_bams, itertools.repeat(nfree_dir), itertools.repeat(return_dict)))
pool.map(function4, itertools.izip(list(return_dict.keys()), itertools.repeat(chrom)))
return_dict = manager.dict()
pool.map(function3, itertools.izip(ddup_bams, itertools.repeat(npres_dir), itertools.repeat(return_dict)))
pool.map(function4, itertools.izip(list(return_dict.keys()), itertools.repeat(chrom)))
def launch_multiprocess(launchpad, fworker, loglvl, nlaunches, num_jobs, sleep_time,
total_node_list=None, ppn=1, timeout=None, exclude_current_node=False,
local_redirect=False):
"""
Launch the jobs in the job packing mode.
Args:
launchpad (LaunchPad)
fworker (FWorker)
loglvl (str): level at which to output logs
nlaunches (int): 0 means 'until completion', -1 or "infinite" means to loop forever
num_jobs(int): number of sub jobs
sleep_time (int): secs to sleep between rapidfire loop iterations
total_node_list ([str]): contents of NODEFILE (doesn't affect execution)
ppn (int): processors per node (doesn't affect execution)
timeout (int): # of seconds after which to stop the rapidfire process
exclude_current_node: Don't use the script launching node as a compute node
local_redirect (bool): redirect standard input and output to local file
"""
# parse node file contents
if exclude_current_node:
host = get_my_host()
l_dir = launchpad.get_logdir() if launchpad else None
l_logger = get_fw_logger('rocket.launcher', l_dir=l_dir, stream_level=loglvl)
if host in total_node_list:
log_multi(l_logger, "Remove the current node \"{}\" from compute node".format(host))
total_node_list.remove(host)
else:
log_multi(l_logger, "The current node is not in the node list, keep the node list as is")
node_lists, sub_nproc_list = split_node_lists(num_jobs, total_node_list, ppn)
# create shared dataserver
ds = DataServer.setup(launchpad)
port = ds.address[1]
manager = Manager()
running_ids_dict = manager.dict()
firing_state_dict = manager.dict()
# launch rapidfire processes
processes = start_rockets(fworker, nlaunches, sleep_time, loglvl, port, node_lists,
sub_nproc_list, timeout=timeout, running_ids_dict=running_ids_dict,
local_redirect=local_redirect, firing_state_dict=firing_state_dict)
FWData().Running_IDs = running_ids_dict
FWData().FiringState = firing_state_dict
# start pinging service
ping_stop = threading.Event()
ping_thread = threading.Thread(target=ping_multilaunch, args=(port, ping_stop))
ping_thread.start()
# wait for completion
for p in processes:
p.join()
ping_stop.set()
ping_thread.join()
ds.shutdown()
def plot_utrs(conditions, rev_conds, outdir):
pool = Pool(24)
manager = Manager()
return_dict = manager.dict()
genes = manager.dict()
pool.map(function1, itertools.izip(list(conditions.keys()), itertools.repeat(outdir), itertools.repeat(return_dict), itertools.repeat(genes)))
combined_profiles = {}
normal = {}
#Have to add all conditions together
today = date.today()
date_format = "{}_{}_{}".format(today.day, today.month, today.year)
pp = PdfPages("{}/{}_UTR_averaged.pdf".format(outdir, date_format))
#First need to average over all genes:
averaged_profiles = {}
len_genes = {}
for key1, key2 in genes.keys(): #Chromosome, bam
if key2 not in len_genes:
len_genes[key2] = {}
len_genes[key2][key1] = 1
else:
len_genes[key2][key1] = 1
for key in return_dict.keys():
averaged_profiles[key] = return_dict[key]/len(len_genes[key].keys())
for key in rev_conds:
fig = pyplot.figure()
pyplot.rc('axes', color_cycle=['b','r', 'c', 'm', 'y', 'k', 'gray', "green"])
for fasta in rev_conds[key]:
name = re.sub(".fa", "", fasta)
name = os.path.basename(name)
uniq_count = read_reports('{}/{}_report.txt'.format(outdir, name))
normal[fasta] = uniq_count
norm = 100000/float(uniq_count)
normalised_profile = norm * averaged_profiles[fasta]
if key not in combined_profiles:
combined_profiles[fasta] = normalised_profile
else:
combined_profiles[fasta] += normalised_profile
pyplot.plot( numpy.arange( -200, 200 ), normalised_profile, label=name)
pyplot.legend(prop={'size':6})
pyplot.title(key)
pp.savefig(fig)
pyplot.close()
fig = pyplot.figure()
pyplot.rc('axes', color_cycle=['b','r', 'c', 'm', 'y', 'k', 'gray', "green"])
for key in combined_profiles:
name = re.sub(".fa", "", key)
name = os.path.basename(name)
pyplot.plot( numpy.arange( -200, 200 ), combined_profiles[key], label=name)
pyplot.legend(prop={'size':6})
pp.savefig(fig)
pp.close()
def __init__(self, **kwargs):
"""
F1 class constructor
:param codi_r1: R1 code of the company
:type codi_r1: str
:param year: Year of the resolution
:type year: int
"""
super(F1, self).__init__(**kwargs)
self.codi_r1 = kwargs.pop('codi_r1')
self.year = kwargs.pop('year', datetime.now().year - 1)
manager = Manager()
self.cts = manager.dict()
self.cnaes = manager.dict()
self.base_object = 'CUPS'
self.report_name = 'F1 - CUPS'
self.reducir_cups = kwargs.get("reducir_cups", False)
mod_all_year = self.connection.GiscedataPolissaModcontractual.search([
("data_inici", "<=", "{}-01-01".format(self.year)),
("data_final", ">=", "{}-12-31".format(self.year))],
0, 0, False, {"active_test": False}
)
mods_ini = self.connection.GiscedataPolissaModcontractual.search(
[("data_inici", ">=", "{}-01-01".format(self.year)),
("data_inici", "<=", "{}-12-31".format(self.year))],
0, 0, False, {"active_test": False}
)
mods_fi = self.connection.GiscedataPolissaModcontractual.search(
[("data_final", ">=", "{}-01-01".format(self.year)),
("data_final", "<=", "{}-12-31".format(self.year))],
0, 0, False, {"active_test": False}
)
self.modcons_in_year = set(mods_fi + mods_ini + mod_all_year)
self.default_o_cod_tfa = None
self.default_o_cnae = None
search_params = [
('name', '=', 'libcnmc_4_2015_default_f1')
]
id_config = self.connection.ResConfig.search(
search_params
)
self.generate_derechos = kwargs.pop("derechos", False)
if id_config:
config = self.connection.ResConfig.read(id_config[0], [])
default_values = literal_eval(config['value'])
if default_values.get('o_cod_tfa'):
self.default_o_cod_tfa = default_values.get('o_cod_tfa')
if default_values.get('o_cnae'):
self.default_o_cnae = default_values.get('o_cnae')
def _get(self, args):
draft_id = args[0]
id = args[1] if len(args) > 1 else None
q = self.db.query(Player)
if id is not None:
player = q.filter(Player.id == int(id)).first()
team = self.db.query(Team).filter(and_(Team.is_owner == True,
Team.draft_id == draft_id)).first()
available_players = self.db.query(Player).join(Player.core).filter(and_(PlayerCore.rank != None,
PlayerCore.target_price != None,
PlayerCore.points > 0,
Player.draft_id == draft_id,
Player.team_id == None,
Player.id != player.id)).order_by(PlayerCore.rank).all()
min_price = 1
max_price = min(player.core.target_price + 21, team.money)
manager = Manager()
max_starters_points = manager.dict()
max_bench_points = manager.dict()
pool = Pool(processes=8)
starters, bench = get_starters_and_bench(self.db, team.id)
max_starters_points[0] = optimizer.optimize_roster(starters, available_players, team.money - (constants.BENCH_SIZE - len(bench)))[1]
for m in range(min_price, 10):
pool.apply_async(wrap_optimizer, args=(starters, available_players, team.money - m - (constants.BENCH_SIZE - len(bench)) + 1, max_bench_points, m))
full_starters = True
for s in starters:
if s is None:
full_starters = False
if not full_starters:
starters_clone = list(starters)
bench_clone = list(bench)
place_player(player, starters_clone, bench_clone)
for m in range(min_price, max_price):
pool.apply_async(wrap_optimizer, args=(starters_clone, available_players, team.money - m - (constants.BENCH_SIZE - len(bench_clone)), max_starters_points, m))
pool.close()
pool.join()
ret = player.to_dict(['core'])
ret['max_starters_points'] = dict(max_starters_points)
ret['max_bench_points'] = dict(max_bench_points)
return ret
else:
players = q.join(PlayerCore).filter(and_(Player.draft_id == int(draft_id),
PlayerCore.rank != None,
PlayerCore.target_price != None)).all()
return {'players': [p.to_dict(['core']) for p in players]}
def __init__(self, **kwargs):
super(CreateCelles, self).__init__(**kwargs)
self.header = [
'name', 'tipus_element', 'installacio', 'tipus_posicio',
'inventari', 'aillament', 'cini', 'propietari', 'perc_financament',
'tensio'
]
self.search_keys = [('name')]
self.fields_read_ct = ['perc_financament', 'propietari']
self.fields_read_at_tram = ['perc_financament']
self.object = self.connection.GiscedataCellesCella
manager = Manager()
self.cts = manager.dict()
self.at_suports = manager.dict()
def parallel_peak_file_plot(ibams, bed_file, size_dict, halfwinwidth, norm, controls):
positions = set()
for line in open(bed_file):
fields = line.split( "\t" )
name = re.sub("chr", "", fields[0])
window = HTSeq.GenomicInterval( name, int(fields[1]), int(fields[2]), "." )
positions.add(window)
if controls == None:
manager = Manager()
return_dict = manager.dict()
pool = Pool(8)
if norm: #Normalisation provided
pool.map(read_bam_function, itertools.izip(list(ibams.keys()), itertools.repeat(positions), itertools.repeat(halfwinwidth),
itertools.repeat(return_dict), itertools.repeat(norm), itertools.repeat(None))) ##Running annotation in parallel
else:
pool.map(read_bam_function, itertools.izip(list(ibams.keys()), itertools.repeat(positions), itertools.repeat(halfwinwidth),
itertools.repeat(return_dict), itertools.repeat(None), itertools.repeat(size_dict)))
pool.close()
pool.join()
for key in return_dict.keys():
pyplot.plot( numpy.arange( -halfwinwidth, halfwinwidth ), return_dict[key], label=ibams[key])
pyplot.legend(prop={'size':8})
pyplot.savefig("Average_peak_profile.pdf".format(ibams[key]))
else:
manager = Manager()
return_dict = manager.dict()
pool = Pool(8)
pool.map(read_bam_function, itertools.izip(list(ibams.keys()), itertools.repeat(positions), itertools.repeat(halfwinwidth),
itertools.repeat(return_dict), itertools.repeat(None), itertools.repeat(None)))
control_dict = manager.dict()
control_bam = []
for key in controls:
control_bam.append(controls[key])
control_sizes = sam_size(control_bam)
pool = Pool(8)
pool.map(read_bam_function, itertools.izip((control_bam), itertools.repeat(positions), itertools.repeat(halfwinwidth),
itertools.repeat(control_dict), itertools.repeat(None), itertools.repeat(None)))
pool.close()
pool.join()
# colors = ["b", "g", "r", "y", "k"] #To make it more robust, just use default colors
c = 0
for key in return_dict.keys():
control = controls[key]
new_profile = return_dict[key] - control_dict[control] #Unsure if working properly, maybe make it more intelligent?
gapdh = read_counts(norm[key])
constant = 1000/float(gapdh)
new_profile = new_profile*constant
pyplot.plot( numpy.arange( -halfwinwidth, halfwinwidth ), new_profile, label=ibams[key])#, color=colors[c])
pyplot.legend(prop={'size':8})
pyplot.savefig("Average_peak_profile.pdf".format(ibams[key]))
def run_parallel(
self, test_suites, test_runner, result_type=None,
results_path=None):
exit_code = 0
proc = None
unittest.installHandler()
processes = []
manager = Manager()
results = manager.dict()
manager.dict()
start = time.time()
test_mapping = {}
for test_suite in test_suites:
# Give each test suite an uuid so it can be
# matched to the correct test result
test_id = str(uuid.uuid4())
test_mapping[test_id] = test_suite
proc = Process(
target=self.execute_test,
args=(test_runner, test_id, test_suite, results))
processes.append(proc)
proc.start()
for proc in processes:
proc.join()
finish = time.time()
errors, failures, _ = self.dump_results(start, finish, results)
if result_type is not None:
all_results = []
for test_id, result in results.items():
tests = test_mapping[test_id]
result_parser = SummarizeResults(
vars(result), tests, (finish - start))
all_results += result_parser.gather_results()
reporter = Reporter(
result_parser=result_parser, all_results=all_results)
reporter.generate_report(
result_type=result_type, path=results_path)
if failures or errors:
exit_code = 1
return exit_code
def controller_switch_failure_test():
s = ["s5"]
clear_config(s)
manager1 = Manager()
manager2 = Manager()
Q1 = manager1.dict()
failure1 = manager1.Value("i", 0)
failed_list1 = manager1.list([])
PID1 = manager1.list(["%02d" % i for i in xrange(1, 51)])
s1 = manager1.list(["s5"])
Q2 = manager2.dict()
failure2 = manager2.Value("i", 0)
failed_list2 = manager2.list([])
PID2 = manager2.list(["%02d" % i for i in xrange(1, 51)])
s2 = manager2.list(["s5"])
processes = []
process1 = mp.Process(target=policy_update, args=(s1, "1", Q1, PID1, failure1, failed_list1))
processes.append(process1)
process2 = mp.Process(target=controller_failure_detection, args=(s1, "1", failure1, failed_list1))
processes.append(process2)
process = mp.Process(target=upon_new_policy, args=(s1, "1", Q1, PID1))
processes.append(process)
process3 = mp.Process(target=policy_update, args=(s2, "2", Q2, PID2, failure2, failed_list2))
processes.append(process3)
process4 = mp.Process(target=controller_failure_detection, args=(s2, "2", failure2, failed_list2))
processes.append(process4)
process5 = mp.Process(target=upon_new_policy, args=(s2, "2", Q2, PID2))
processes.append(process5)
# Run processes
for p in processes:
p.start()
print "STARTING:", p, p.is_alive()
time.sleep(10)
print "terminated"
processes[3].terminate()
processes[4].terminate()
print "sleeping"
time.sleep(5)
processes[5].terminate()
print "terminated 5"
time.sleep(2)
for p in processes:
print "TERMINATED:", p, p.is_alive()
for p in processes:
p.join()
print "JOINED:", p, p.is_alive()
def main(data):
"""
Launches the server with given set of keys (data)
"""
#this library needs to be replaced
ned = rsa.newKey(10**100,10**101,50)
print "Keys\nn = %s\ne = %s\nd = %s\n" % (ned[0],ned[1],ned[2])
#create thread safe dictionaries
manager = Manager()
ID_KEY = manager.dict()
ID_KEY.update(data) #get saved data
ID_STATUS = manager.dict()
ID_SOCK = manager.dict()
#define an interupt catcher to save data
def signal_handler(signal, frame):
print('Saving data')
#this is ugly but the only reasonable way I could find
#to unroll a managed threadsafe dictionary to a normal one
#that can interact with pickle correctly
with lock:
with open('serverdata.pkl', 'wb') as f:
temp = ast.literal_eval(str(ID_KEY))
pickle.dump( temp, f ) #save the keys
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
#listen on port 8000 (probably a bad choice, usually used for testing web servers) for incoming connections
s = socket.socket()
hostname = socket.gethostname()
port = 8000
s.bind(('',port))
s.listen(5)
#listen for incoming connections and start a thread based on what kind it is
while True:
c, addr = s.accept()
c.send("init")
v = c.recv(4)
if "user" == v:
#incomming connection is a user which will send queries
Process(target=process_client, args=(c,ID_KEY,ID_STATUS,ID_SOCK, ned)).start()
elif "circ" == v:
#incoming connection is to be used as part of a circuit
Process(target=process_circuit, args=(c,ID_KEY,ID_STATUS,ID_SOCK,ned)).start()
elif "data" == v:
#incoming connection has some data for the server to process
Process(target=process_data, args=(c,ID_SOCK,ned)).start()
def search(self, links=False):
"""
Get links from the search engines and fill them to the respective lists.
It gets self.pages of links from Search Engines, sends them to the formatter functions and gets the lists.
:return: nothing
"""
if self.type == "text":
mg = Manager()
ret = mg.dict()
jobs = []
p1 = Process(target=self.google_proc, args=(ret,))
jobs.append(p1)
p2 = Process(target=self.yahoo_proc, args=(ret,))
jobs.append(p2)
p3 = Process(target=self.bing_proc, args=(ret,))
jobs.append(p3)
p1.start()
p2.start()
p3.start()
for proc in jobs:
proc.join()
temp = ret.values()[0] + ret.values()[1] + ret.values()[2]
print temp
for i in temp:
f = 0
for j in self.uniquelinks:
if i[1] == j[1]:
f = 1
if f == 0:
self.uniquelinks.append(i)
if links:
return self.uniquelinks
else: # [[title, link, data], [title, link, data] ...]
mg = Manager()
ret = mg.dict()
jobs = []
n = 0
for li in self.uniquelinks[0:3]:
p = Process(target=self.data_collector, args=(n, li[1], ret))
n += 1
jobs.append(p)
p.start()
for proc in jobs:
proc.join()
print ret.values()
print len(ret.values())
def build_all_indexes(page_titles, wiki_index, process_limit=4):
"""
This function runs link_indexer and token_indexer together in a single loop,
instead of making one (costly!) pass for each using seperate functions.
"""
manager = Manager()
link_index = manager.dict()
token_index = manager.dict()
lock = manager.Lock()
page_titles = [i.lower() for i in page_titles]
run(page_titles, all_indexer_instance, (wiki_index, link_index, token_index, lock), process_limit=process_limit)
return {'link_index': link_index, 'token_index': token_index}
def get_hls_stream(m3u8_url, concurrency=1, live=True, loop=1, segment_sleep=1, authentication=None, timeouts=None):
# Spawn concurrent subprocesses to get every HLS segment of stream
# Disable all SSL Warnings (version dependent)
try:
requests.packages.urllib3.disable_warnings()
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
except:
pass
# Configurables
subprocesses = []
process_id = 0
timeout = {'read': float(timeouts['read']),
'connect': float(timeouts['connect']),
'sleep': float(segment_sleep)}
manager = Manager()
lock = manager.Lock()
durations = manager.list()
success = manager.dict()
results = manager.dict()
status = manager.dict()
playlists = manager.dict()
# Cookies for session authentication
if authentication:
cookies = (authentication['url'], authentication['username'], authentication['password'], authentication['type'])
else:
cookies = None
# Spawn parallel subprocesses for each simulated client
for x in range(0, int(concurrency)):
process_id += 1
p = Process(target=get_playlist, args=(m3u8_url, live, loop, results, status, success, durations, playlists, timeout, cookies, lock, process_id,))
subprocesses.append(p)
p.start()
# Wait for all processes to complete
for subprocess in subprocesses:
while True:
response_times = calculated_response_times(durations)
yield results._getvalue(), status._getvalue(), response_times, success._getvalue()
time.sleep(1)
if not subprocess.is_alive():
yield results._getvalue(), status._getvalue(), response_times, success._getvalue()
break
def scanner_network(self,gateway):
scan = ''
config_gateway = gateway.split('.')
del config_gateway[-1]
for i in config_gateway:
scan += str(i) + '.'
gateway = scan
ranger = str(self.ip_range.text()).split('-')
jobs = []
manager = Manager()
on_ips = manager.dict()
for n in xrange(int(ranger[0]),int(ranger[1])):
ip='%s{0}'.format(n)%(gateway)
p = Process(target=self.working,args=(ip,on_ips))
jobs.append(p)
p.start()
for i in jobs: i.join()
for i in on_ips.values():
Headers = []
n = i.split('|')
self.data['IPaddress'].append(n[0])
self.data['MacAddress'].append(n[1])
self.data['Hostname'].append('<unknown>')
for n, key in enumerate(reversed(self.data.keys())):
Headers.append(key)
for m, item in enumerate(self.data[key]):
item = QTableWidgetItem(item)
item.setTextAlignment(Qt.AlignVCenter | Qt.AlignCenter)
self.tables.setItem(m, n, item)
Headers = []
for key in reversed(self.data.keys()):
Headers.append(key)
self.tables.setHorizontalHeaderLabels(Headers)
def main():
init_params()
vk = connect_to_vk(LOGIN, PASSWORD)
audio_list = vk.method('audio.get', {})
total = len(audio_list)
if not os.path.exists(DOWNLOAD_DIR):
os.makedirs(DOWNLOAD_DIR)
manager = Manager()
workers_list = []
progress_list = manager.dict()
downloaded_tracks = manager.Value('i', 0)
lock = Lock()
for f in audio_list[:WORKERS_COUNT - 1]:
start_download_process(f, workers_list, progress_list, downloaded_tracks, lock)
del audio_list[:WORKERS_COUNT - 1]
while any(worker.is_alive() for worker in workers_list) or len(audio_list):
if audio_list and len(workers_list) < WORKERS_COUNT:
f = audio_list.pop(0)
start_download_process(f, workers_list, progress_list, downloaded_tracks, lock)
print_progress(progress_list, downloaded_tracks.value, total)
clean_workers(workers_list)
time.sleep(0.1)
print "Done!"
def getData():
if os.path.isfile("chat_urls.p"):
chat_urls = pickle.load( open( "chat_urls.p", "rb" ) )
else:
chat_urls = {}
for user in users:
chat_urls[user] = get_urls(user)
teams_url = "http://espn.go.com/mlb/teams"
pickle.dump( chat_urls, open( "chat_urls.p", "wb" ) )
# for user in chat_urls:
# urls = chat_urls[user]
# for url in urls:
# getLog(url)
logDB = {}
for user in chat_urls:
logDB[user] = {}
p = Pool(20)
i=0
manager = Manager()
db = manager.dict()
for user in chat_urls:
for url in chat_urls[user]:
i+=1
p.apply_async(addLogData, args=(url,db))
p.close()
p.join()
out = db._getvalue()
outfile = open("rawChat.txt","wb")
for url in out:
outfile.write(out[url]+"\n")
def __init__(self, firefox=None, email=None, senha=None, pasta=None):
"""'firefox' é o caminho para o binário do Firefox a ser usado.
'pasta' é o caminho para a pasta onde salvar os downloads."""
self.firefox = firefox
self.pasta = pasta
self.email = email
self.senha = senha
self.navegador = None
self.app = None
self.logger = None
manager = Manager()
self.safe_dict = manager.dict()
self.clear_captcha()
self.stop()
self.try_break_audio_captcha = True
self.nome_audio_captcha = "somCaptcha.wav"
self.recognizer = sr.Recognizer(str('pt-BR'))
self.user_agent = (
"User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:28.0)"
" Gecko/20100101 Firefox/28.0"
)
self.base_url = 'http://esic.prefeitura.sp.gov.br'
self.login_url = self.base_url + '/Account/Login.aspx'
self.logado = False
self.ja_tentou_cookies_salvos = False
self.rodar_apenas_uma_vez = False
def concurrent_test(robot, rooms, num_trials, start_location = -1, chromosome = None):
"""
Run the tests in multiple processes. Can be directly swapped out for testAllMaps.
"""
# Setup variables
num_rooms = len(rooms) # Total number of rooms
total_trials = num_trials * num_rooms # Total number of trials
processes = [] # List for all processes
manager = Manager() # Manager to handle result transfer
dict = manager.dict() # Dict which will store results
# Create a process for each room, storing parameters in instance variables
for i, room in enumerate(rooms):
process = SimulationProcess(i, dict)
process.robot = robot
process.room = room
process.num_trials = num_trials
process.start_location = start_location
process.chromosome = chromosome
process.start()
processes.append(process)
#end for
# Print the results
total_score = 0
for i, process in enumerate(processes):
process.join()
(score, std) = dict[i]
print("Room %d of %d done (score: %d std: %d)" % (i + 1, num_rooms, score, std))
total_score += score
#end for
print("Average score over %d trials: %d" % (total_trials, total_score / num_rooms))
return total_score / num_rooms
#end concurrent_test
def run(args):
# Limit it to a single GPU.
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu)
conn = create_db(args.db)
m = Manager()
logs = args.logging
datasets = args.datasets
embeddings = args.embeddings
settings = args.settings
# So we don't litter the fs
dir_ = tempfile.mkdtemp(prefix='baseline-speed-test-')
try:
configs = get_configs(args.config)
if not args.single:
full_configs = []
for config in configs:
full_configs.extend(edit_config(config, args.frameworks, args.no_crf, args.no_attn))
configs = full_configs
if args.verbose:
for config in configs:
pprint(config)
print()
print()
steps = len(configs)
pg = create_progress_bar(steps)
for config in configs:
write_config = deepcopy(config)
config['train']['epochs'] = args.trials
task_name = config['task']
system_info = m.dict()
p = Process(
target=run_model,
args=(
system_info,
config,
logs,
settings,
datasets,
embeddings,
task_name,
dir_,
int(args.gpu)
)
)
p.start()
pid = p.pid
p.join()
log_file = os.path.join(dir_, 'timing-{}.log'.format(pid))
speeds = parse_logs(log_file)
save_data(conn, speeds, write_config, system_info)
pg.update()
pg.done()
finally:
shutil.rmtree(dir_)
def func_thread():
a = numpy.random.rand(1000000)
b = numpy.random.rand(1000000)
nodata = 0.3
print "here"
manager = Manager()
lock = Lock()
d = manager.dict()
ps = []
start_time = time.clock()
for i in numpy.where((a > 0.7) & (a < 0.9) & (a != nodata)):
for j in numpy.where((b > 0.5) & (b < 0.9) & (b != nodata)):
index = numpy.intersect1d(i, j)
length = len(index)/2
array1 = index[:length]
array2 = index[length:]
for processes in range(2):
p = Process(target=f_thread, args=(d, a, b, array1, lock))
ps.append(p)
p.start()
for p in ps:
p.join()
print time.clock() - start_time, "seconds"
print len(d)
def __init__(self,port):
manager = Manager()
self.status=manager.dict()
self.sendbuf=manager.list()
self.p = Process(target=SocketManager, args=(port,self.status,self.sendbuf) )
self.p.daemon=True
self.p.start()
def sync():
from multiprocessing import Manager
from common import bounty, settings, peers
from common.safeprint import safeprint
man = Manager()
items = {'config':man.dict(),
'peerList':man.list(),
'bountyList':man.list(),
'bountyLock':bounty.bountyLock,
'keyList':man.list()}
items['config'].update(settings.config)
items['peerList'].extend(peers.peerlist)
items['bountyList'].extend(bounty.bountyList)
safeprint(items)
safeprint(items.get('bountyList'))
safeprint(items.get('keyList'))
if items.get('config') is not None:
from common import settings
settings.config = items.get('config')
if items.get('peerList') is not None:
global peerList
peers.peerlist = items.get('peerList')
if items.get('bountyList') is not None:
from common import bounty
bounty.bountyList = items.get('bountyList')
if items.get('bountyLock') is not None:
from common import bounty
bounty.bountyLock = items.get('bountyLock')
return items
def main():
if len(sys.argv) > 1:
print "cmd arg to set directory to: " + sys.argv[1]
os.chdir(sys.argv[1])
print "cwd is: " + os.getcwd()
# make sure we have the correct device
keepTrying = True
countCurrent = 0
countCurrentFail = 0
manager = Manager()
sharedDictionary = manager.dict()
while keepTrying:
serial0 = serial.Serial("/dev/ttyACM0") # connection to arduino1
serial1 = serial.Serial("/dev/ttyACM1") # connection to arduino2
try:
line = ser.readline() # read ardiono about once every two seconds
I = float(line.split(" ")[1].strip()) # get the current reading
countCurrent += 1
except Exception, e:
countCurrentFail += 1
if countCurrent > countCurrentFail + 5: # 5 good readings
keepTrying = False
startThreading(sharedDictionary, serial0, serial1)
elif countCurrentFail > countCurrent + 5: # 5 bad readings, do a swap
keepTrying = False
startThreading(sharedDictionary, serial1, serial0)
print " . " + str(countCurrent) + "-" + str(countCurrentFail)
def processFiles(patch_dir):
root = os.getcwd()
glbl.data_dirs = {}
if root != patch_dir: working_path = root+"/"+patch_dir
else: working_path = root
for path, dirs, files in os.walk(working_path):
if len(dirs) == 0: glbl.data_dirs[path] = ''
# Multiprocessing Section
#########################################
Qids = glbl.data_dirs.keys()
manager = Manager() # creates shared memory manager object
results = manager.dict() # Add dictionary to manager, so it can be accessed across processes
nextid = Queue() # Create Queue object to serve as shared id generator across processes
for qid in Qids: nextid.put(qid) # Load the ids to be tested into the Queue
for x in range(0,multiprocessing.cpu_count()): # Create one process per logical CPU
p = Process(target=processData, args=(nextid,results)) # Assign process to processCBR function, passing in the Queue and shared dictionary
glbl.jobs.append(p) # Add the process to a list of running processes
p.start() # Start process running
for j in glbl.jobs:
j.join() # For each process, join them back to main, blocking on each one until finished
# write out results
c = 1
sets = results.keys()
sets.sort()
for x in sets:
if results[x] != 'None':
FINAL = open('result'+str(c)+'.txt','w')
n = "\n************************************************************************************************\n"
FINAL.write(n+"* "+x+' *\n'+n+results[x]+"\n")
FINAL.close()
c += 1
class Airscan():
_instance = None
def __init__(self):
self.manager = Manager()
self.aps = self.manager.list()
self.asoc = self.manager.list()
self.nasoc = self.manager.list()
self.flags = self.manager.dict()
def packet_handler(self, pkt):
if pkt.haslayer(Dot11):
Dot11Layer = pkt.getlayer(Dot11)
if Dot11Layer.addr1 == Dot11Layer.addr3 and Dot11Layer.addr1 != 'ff:ff:ff:ff:ff:ff':
client = "%s %s" % (Dot11Layer.addr1, Dot11Layer.addr2)
if client not in self.asoc:
self.asoc.append(client)
print "%s%s %s%s" % (RED, Dot11Layer.addr1,
Dot11Layer.addr2, END)
if pkt.haslayer(Dot11ProbeReq):
Dot11ProbeReqLayer = pkt.getlayer(Dot11ProbeReq)
if len(Dot11ProbeReqLayer.info) > 0:
client = "%s %s" % (Dot11Layer.addr2,
Dot11ProbeReqLayer.info)
if client not in self.nasoc:
self.nasoc.append(client)
print "%s%s %s%s" % (YEL, Dot11Layer.addr2,
Dot11ProbeReqLayer.info, END)
if pkt.haslayer(Dot11Beacon):
Dot11BeaconLayer = pkt.getlayer(Dot11Beacon)
if Dot11Layer.addr2 and (Dot11Layer.addr2 not in self.aps):
self.aps.append(Dot11Layer.addr2)
channel = int(ord(pkt[Dot11Elt:3].info))
cap = pkt.sprintf(
"{Dot11Beacon:%Dot11Beacon.cap%}\{Dot11ProbeResp:%Dot11ProbeResp.cap%}"
)
if re.search("privacy", cap):
enc = 'Y'
else:
enc = 'N'
rssi = -(256 - ord(pkt.notdecoded[-4:-3]))
print "%s%d %s %d %s %s%s" % (BLU, channel, enc, rssi,
Dot11Layer.addr2,
Dot11BeaconLayer.info, END)
def keep_handler(self, pkt):
return self.flags['stop_sniff']
def channel_hop(self):
for channel in range(1, 14):
os.system("iw dev %s set channel %d" % (self.interface, channel))
time.sleep(0.7)
self.flags['stop_sniff'] = True
self.aps[:] = []
self.asoc[:] = []
self.nasoc[:] = []
def signal_handler(self, signal, frame):
self.flags['stop_sniff'] = True
self.p.terminate()
self.p.join()
def run(self, iface):
self.interface = iface
self.flags['stop_sniff'] = False
self.p = Process(target=self.channel_hop)
self.p.start()
signal.signal(signal.SIGINT, self.signal_handler)
sniff(iface=self.interface,
prn=self.packet_handler,
stop_filter=self.keep_handler)
self.p.terminate()
self.p.join()
target = raw_input("Enter target bssid fully or partially:")
airjam = Airjam(self.interface, target)
airjam.run()
def getInstance():
if Airscan._instance == None:
Airscan._instance = Airscan()
return Airscan._instance
getInstance = staticmethod(getInstance)
from multiprocessing import Manager
def set_exit_handler(func):
signal.signal(signal.SIGTERM, func)
def on_exit(sig, func=None):
print("exit handler triggered")
sys.exit(1)
if __name__ == '__main__':
set_exit_handler(on_exit)
options = None
# grab options, setup
envArgs = os.getenv("ARELLE_ARGS")
manager = Manager()
output = manager.dict()
if envArgs:
args = shlex.split(envArgs)
else:
args = sys.argv[1:]
try:
numthreads = int(args[args.index('--xule-numthreads')+1])
except ValueError:
numthreads = 1
gettext.install("arelle") # needed for options messages
print("Initializing Server")
cntlr = CntlrCmdLine.parseAndRun(args)
cntlr.startLogging(logFileName='logToBuffer')
# get generated options from controller
from multiprocessing import Manager
from threading import Lock
from typing import Any, Iterable, Optional
manager = Manager()
shared_memory: dict[str, Any] = manager.dict()
lock: Lock = manager.Lock()
class Db:
_cache: dict[str, Any]
_lock: Lock
def __init__(self) -> None:
self._cache = shared_memory
self._lock = lock
@property
def cache(self) -> dict[str, Any]:
return self._cache
def connect(self) -> dict[str, Any]:
return self.cache
def close(self) -> None:
self.clear()
def get(self, key: str) -> Optional[Any]:
return self.cache.get(key, None)
def all(self) -> Iterable[Any]:
y_predict=cross_val_predict(svm.SVC(kernel='rbf',C=C,gamma=gamma,),X_train,y_train,cv=cross_validation_value,n_jobs=CPU_value)
y_predict_prob=cross_val_predict(svm.SVC(kernel='rbf',C=C,gamma=gamma,probability=True),X_train,y_train,cv=cross_validation_value,n_jobs=CPU_value,method='predict_proba')
input_file = input_file.replace(".csv","")
y_predict_path = input_file + "_predict.csv"
y_predict_proba_path = input_file + "_predict_proba.csv"
share_y_predict_dict[input_file] = y_predict
share_y_predict_proba_dict[input_file] = y_predict_prob[:,1]
pd.DataFrame(y_predict).to_csv(y_predict_path, header = None, index = False)
pd.DataFrame(y_predict_prob[:,1]).to_csv(y_predict_proba_path, header = None, index = False)
print("子进程终止>>> pid={0}".format(os.getpid()))
if __name__=="__main__":
print("主进程执行中>>> pid={0}".format(os.getpid()))
manager = Manager()
share_y_predict_dict = manager.dict()
share_y_predict_proba_dict = manager.dict()
ps=[]
if default_l == 1:
data = ""
x_len = 1000
y_len = 1000
file_len = len(input_files)
threshold = file_len/2
for index, input_file in enumerate(input_files):
data = pd.read_csv(input_file,header=None)
(x_len,y_len) = data.shape
X_train = data.iloc[:,0:y_len-1]
y_train = data.iloc[:,[y_len-1]]
X_train = X_train.values
from playback import *
from volume import *
def setup_pins():
GPIO.setmode(GPIO.BCM)
GPIO.setup(17, GPIO.IN, pull_up_down=GPIO.PUD_UP)
def reset_recordings(channel):
global recordings
metronome(recordings)
if __name__ == "__main__":
manager = Manager()
# Initialize shared variables for recordings and volumes
recordings = manager.dict()
volumes = manager.dict()
# Metronome initializes the recordings with a metronome on channel 7
metronome_proc = Process(target=metronome, args=(recordings,))
metronome_proc.start()
metronome_proc.join()
# Sets volumes to 127 initially
setVolumes = Process(target=set_volumes, args=(volumes,))
setVolumes.start()
setVolumes.join()
setup_pins()
GPIO.add_event_detect(17, GPIO.FALLING, callback=reset_recordings, bouncetime=300)
def convert_html_to_pdf(template_file, pdf_name, dirName, dirName_Output):
pdfkit.from_file(os.path.join(os.path.join(dirName, template_file)),
os.path.join(os.path.join(dirName_supp, pdf_name)),
options=options_supp)
############################################################################################################################
# Run script
#################################################
if __name__ == "__main__":
utility.clean_all()
createdirs(dirNames)
manager = Manager() # create only 1 mgr
d = manager.dict() # create only 1 dict
report = WriteReport(args.f)
template_dict = report.run_entry_composition(Template_Dict)
template_dict, clashscore, rama, sidechain, exv_data = report.run_model_quality(
template_dict)
template_dict, sas_data, sas_fit = report.run_sas_validation(template_dict)
cx_fit, template_dict = report.run_cx_validation(template_dict)
report.run_quality_glance(clashscore, rama, sidechain, exv_data, sas_data,
sas_fit, cx_fit)
report.run_sas_validation_plots(template_dict)
write_pdf(args.f, template_dict, template_pdf, dirNames['pdf'],
dirNames['pdf'])
template_dict = report.run_supplementary_table(template_dict,
location=args.ls,
physics=physics,
method_details=args.m,
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from multiprocessing import Process
from multiprocessing import Manager
import time
def f1(i, dic):
dic[i] = 200 + i
time.sleep(i)
print(dic.values())
if __name__ == '__main__': #进程间默认不能共用内存
manager = Manager()
dic = manager.dict() #这是一个特殊的字典
for i in range(10):
p = Process(target=f1, args=(i, dic))
p.start()
p.join()
def add_routes(app):
app.router.add_static('/static', path=PROJECT_ROOT)
app.router.add_get('/remote', store_handlers.get_remote_manifest)
app.router.add_get('/local', store_handlers.get_local_manifest)
app.router.add_get('/modules/{module}/{version}/readme',
store_handlers.get_module_readme)
app.router.add_post('/install', install_module)
app.router.add_post('/uninstall', store_handlers.uninstall_module)
app.router.add_get('/installstream', get_install_stream)
if __name__ == '__main__':
manager = Manager()
INSTALL_STATE = manager.dict()
SSE_UPDATE_CONDITION = manager.Condition()
SSE_UPDATE_EVENT = manager.Event()
INSTALL_STATE['stage'] = ''
INSTALL_STATE['message'] = ''
INSTALL_STATE['module_name'] = ''
INSTALL_STATE['module_version'] = ''
INSTALL_STATE['cur_chunk'] = 0
INSTALL_STATE['total_chunks'] = 0
INSTALL_STATE['cur_size'] = 0
INSTALL_STATE['total_size'] = 0
INSTALL_STATE['update_time'] = time.time()
install_worker = Process(target=install_from_queue,
args=(INSTALL_QUEUE, INSTALL_STATE,
SSE_UPDATE_EVENT))
install_worker.start()
def main():
args = parse_arguments()
manager = Manager()
performance_data = manager.dict()
with open(args.test_config) as test_config:
test_conf = json.loads(test_config.read())
_, file_name = ntpath.split(args.test_config)
measure_id, _ = os.path.splitext(file_name)
log.info("Starting performance tests for: " + measure_id)
# block processes before measuring queries
# wait for all processes to store generated results
store_done = multiprocessing.Event()
wait = test_conf.get("wait_for_store", False)
if wait:
store_done.set()
proc_done_counter = Value('i', args.job_num)
proc_counter_lock = Lock()
procs = [
Process(target=measure,
args=(test_conf, performance_data, store_done,
proc_done_counter, proc_counter_lock, args.keep))
for i in range(args.job_num)
]
for proc in procs:
proc.start()
log.info("Measuring report storage time.")
if wait:
while proc_done_counter.value != 0:
time.sleep(1)
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('\n')
log.info("Measuring queries ...")
store_done.set()
for proc in procs:
proc.join()
log.info("Measuring queries done.")
if not test_conf.get("clean_after_fill", True) and not args.keep:
view_host, view_port = get_viewer_host_port(test_conf)
del_procs = [
Process(target=delete_results,
args=(view_host, view_port, run_id, performance_data))
for run_id in performance_data.keys()
]
for proc in del_procs:
proc.start()
for proc in del_procs:
proc.join()
if not len(performance_data):
log.info("There are no measurements results")
sys.exit(0)
file_name = measure_id + '.csv'
if not os.path.exists(args.output):
os.makedirs(args.output)
output = os.path.join(args.output, file_name)
with open(output, 'w+') as out:
process_perf_data(performance_data, out)
log.info('Measurement results:\n' + str(output))
def download(self, project, directory_path, is_forced):
process_results = []
success_data_count = 0
success_labels_count = 0
data_error_results = {}
label_error_results = {}
# get data count
main_label_count, labels = self.label_manager.get_labels(
project.id, label_type='DEFAULT', page=1, page_size=1)
# get labels count
total_label_count, labels = self.label_manager.get_labels(project.id,
page=1,
page_size=1)
#Download project configuration
try:
project_config_path = os.path.join(directory_path, 'project.json')
with open(project_config_path, 'w') as input:
json.dump(project.label_interface, input, indent=4)
is_download_project_config = True
except Exception as e:
is_download_project_config = False
if main_label_count != 0:
page_length = int(
main_label_count / LABEL_DESCRIBE_PAGE_SIZE
) if main_label_count % LABEL_DESCRIBE_PAGE_SIZE == 0 else int(
main_label_count / LABEL_DESCRIBE_PAGE_SIZE) + 1
if not is_forced:
if not click.confirm(
f"Downloading {main_label_count} data and {total_label_count} labels from project '{project.name}' to '{directory_path}'. Proceed?"
):
return
manager = Manager()
label_results_dict = manager.dict()
label_results = manager.list([label_results_dict] * page_length)
process_results = manager.list(range(page_length))
with Pool(NUM_MULTI_PROCESS) as p:
list(
tqdm.tqdm(p.imap(
_download_worker,
zip([self.label_manager] * page_length,
[project.id] * page_length, range(page_length),
[directory_path] * page_length, label_results,
[process_results] * page_length)),
total=page_length))
for key in label_results_dict.keys():
results_error = label_results_dict[key]['error']
success_data_count += 1
success_labels_count += label_results_dict[key][
'success_labels_count']
if 'data' in results_error:
data_error_results[key] = results_error['data']
if 'label' in results_error:
label_error_results[key] = results_error['label']
console.print('\n[b blue]** Result Summary **[/b blue]')
console.print(
f'Download of project configuration - {"[b blue]Success[/b blue]" if is_download_project_config else "[b red]Fail[/b red]"}'
)
console.print(
f'Successful download of {success_labels_count} out of {total_label_count} labels. ({round(success_labels_count/total_label_count*100,2)}%) - [b red]{total_label_count - success_labels_count} ERRORS[/b red]'
)
console.print(
f'Successful download of {success_data_count} out of {main_label_count} data. ({round(success_data_count/main_label_count*100,2)}%) - [b red]{main_label_count - success_data_count} ERRORS[/b red]'
)
# TODO: Need to refactor Get Labels API Fail Logic
for process_result in process_results:
is_process_fail = False
if not process_result:
is_process_fail = True
if is_process_fail:
console.print(
f'[b red]Failed Download Labels from API. Please retry download.[/b red]'
)
self._print_error_table(
data_error_results=data_error_results,
label_error_results=label_error_results,
)
def upload_data(self, project, dataset_name, directory_path, include_label,
is_forced):
imgs_path = recursive_glob_image_files(directory_path)
if not is_forced:
if not click.confirm(
f"Uploading {len(imgs_path)} data and {len(recursive_glob_label_files(directory_path)) if include_label else 0 } labels to dataset '{dataset_name}' under project '{project.name}'. Proceed?"
):
return
asset_images = []
manager = Manager()
if len(imgs_path) != 0:
for key in imgs_path:
file_name = key
asset_image = {
'file': imgs_path[key],
'file_name': file_name,
'data_key': key,
'dataset': dataset_name
}
asset_images.append(asset_image)
data_results = manager.list([manager.dict()] * len(asset_images))
console.print(f"Uploading data:")
with Pool(NUM_MULTI_PROCESS) as p:
list(
tqdm.tqdm(p.imap(
_upload_asset,
zip([project.id] * len(asset_images), asset_images,
data_results)),
total=len(asset_images)))
else:
data_results = [{}]
label_results = None
if include_label:
labels_path = recursive_glob_label_files(directory_path)
console.print(f"Uploading labels:")
if len(labels_path) != 0:
label_results = manager.list([manager.dict()] *
len(labels_path))
with Pool(NUM_MULTI_PROCESS) as p:
list(
tqdm.tqdm(p.imap(
_update_label,
zip([self.label_manager] * len(labels_path),
labels_path, [project.id] * len(labels_path),
[project.label_interface] * len(labels_path),
[dataset_name] * len(labels_path),
label_results)),
total=len(labels_path)))
else:
label_results = [{}]
console.print('\n[b blue]** Result Summary **[/b blue]')
success_data_count = len(asset_images) - len(data_results[0])
data_success_ratio = round(
success_data_count / len(asset_images) *
100, 2) if len(data_results[0]) != 0 else 100
console.print(
f'Successful upload of {success_data_count} out of {len(asset_images)} data. ({data_success_ratio}%) - [b red]{len(data_results[0])} ERRORS[/b red]'
)
if include_label:
success_label_count = len(labels_path) - len(label_results[0])
label_success_ratio = round(
success_label_count / len(labels_path) *
100, 2) if len(label_results[0]) != 0 else 100
console.print(
f'Successful upload of {success_label_count} out of {len(labels_path)} labels. ({label_success_ratio}%) - [b red]{len(label_results[0])} ERRORS[/b red]'
)
self._print_error_table(data_error_results=dict(data_results[0]),
label_error_results=dict(label_results[0]))
else:
self._print_error_table(data_error_results=dict(data_results[0]))
def generatePathwaysList(self):
"""
This function gets a list of selected compounds and the list of matched genes and
find out all the pathways which contain at least one feature.
@param {type}
@returns
"""
from multiprocessing import Process, cpu_count, Manager
from math import ceil
#****************************************************************
# Step 1. GET THE KEGG DATA AND PREPARE VARIABLES
#****************************************************************
inputGenes = self.getInputGenesData().values()
inputCompounds = self.getInputCompoundsData().values()
pathwayIDsList = KeggInformationManager().getAllPathwaysByOrganism(self.getOrganism())
#GET THE IDS FOR ALL PATHWAYS FOR CURRENT SPECIE
enrichmentByOmic = {x.get("omicName"): x.get("featureEnrichment", False) for x in self.getGeneBasedInputOmics() + self.getCompoundBasedInputOmics()}
totalFeaturesByOmic, totalRelevantFeaturesByOmic = self.calculateTotalFeaturesByOmic(enrichmentByOmic)
totalInputMatchedCompounds = len(self.getInputCompoundsData())
totalInputMatchedGenes = len(self.getInputGenesData())
totalKeggPathways = len(pathwayIDsList)
mappedRatiosByOmic = self.getMappedRatios()
#****************************************************************
# Step 2. FOR EACH PATHWAY OF THE SPECIE, CHECK IF THERE IS ONE OR
# MORE FEATURES FROM THE INPUT (USING MULTITHREADING)
#****************************************************************
# try:
# #CALCULATE NUMBER OF THREADS
# nThreads = min(cpu_count(), MAX_THREADS)
# except NotImplementedError as ex:
# nThreads = MAX_THREADS
nThreads = MAX_THREADS
logging.info("USING " + str(nThreads) + " THREADS")
def matchPathways(jobInstance, pathwaysList, inputGenes, inputCompounds, totalFeaturesByOmic, totalRelevantFeaturesByOmic, matchedPathways, mappedRatiosByOmic, enrichmentByOmic):
#****************************************************************
# Step 2.1. FOR EACH PATHWAY IN THE LIST, GET ALL FEATURE IDS
# AND CALCULATE THE SIGNIFICANCE FOR THE PATHWAY
#****************************************************************
keggInformationManager = KeggInformationManager()
genesInPathway = compoundsInPathway = pathway = None
for pathwayID in pathwaysList:
genesInPathway, compoundsInPathway = keggInformationManager.getAllFeatureIDsByPathwayID(jobInstance.getOrganism(), pathwayID)
isValidPathway, pathway = self.testPathwaySignificance(genesInPathway, compoundsInPathway, inputGenes, inputCompounds, totalFeaturesByOmic, totalRelevantFeaturesByOmic, mappedRatiosByOmic, enrichmentByOmic)
if(isValidPathway):
pathway.setID(pathwayID)
pathway.setName(keggInformationManager.getPathwayNameByID(jobInstance.getOrganism(), pathwayID))
pathway.setClassification(keggInformationManager.getPathwayClassificationByID(jobInstance.getOrganism(), pathwayID))
pathway.setSource(keggInformationManager.getPathwaySourceByID(jobInstance.getOrganism(), pathwayID))
matchedPathways[pathwayID] = pathway
manager=Manager()
matchedPathways=manager.dict() #WILL STORE THE OUTPUT FROM THE THREADS
nPathwaysPerThread = int(ceil(len(pathwayIDsList)/nThreads)) + 1 #GET THE NUMBER OF PATHWAYS TO BE PROCESSED PER THREAD
pathwaysListParts = chunks(pathwayIDsList, nPathwaysPerThread) #SPLIT THE ARRAY IN n PARTS
threadsList = []
#LAUNCH THE THREADS
for pathwayIDsList in pathwaysListParts:
thread = Process(target=matchPathways, args=(self, pathwayIDsList, inputGenes, inputCompounds, totalFeaturesByOmic, totalRelevantFeaturesByOmic, matchedPathways, mappedRatiosByOmic, enrichmentByOmic))
threadsList.append(thread)
thread.start()
#WAIT UNTIL ALL THREADS FINISH
for thread in threadsList:
thread.join(MAX_WAIT_THREADS)
isFinished = True
for thread in threadsList:
if(thread.is_alive()):
isFinished = False
thread.terminate()
logging.info("THREAD TERMINATED IN generatePathwaysList")
if not isFinished:
raise Exception('Your data took too long to process and it was killed. Try it again later or upload smaller files if it persists.')
self.setMatchedPathways(dict(matchedPathways))
totalMatchedKeggPathways=len(self.getMatchedPathways())
# Get the adjusted p-values (they need to be passed as a whole)
pvalues_list = defaultdict(dict)
combined_pvalues_list = defaultdict(dict)
for pathway_id, pathway in self.getMatchedPathways().iteritems():
for omic, pvalue in pathway.getSignificanceValues().iteritems():
pvalues_list[omic][pathway_id] = pvalue[2]
for method, combined_pvalue in pathway.getCombinedSignificancePvalues().iteritems():
combined_pvalues_list[method][pathway_id] = combined_pvalue
adjusted_pvalues = {omic: adjustPvalues(omicPvalues) for omic, omicPvalues in pvalues_list.iteritems()}
adjusted_combined_pvalues = {method: adjustPvalues(methodCombinedPvalues) for method, methodCombinedPvalues in combined_pvalues_list.iteritems()}
# Set the adjusted p-value on a pathway basis
for pathway_id, pathway in self.getMatchedPathways().iteritems():
for omic, pvalue in pathway.getSignificanceValues().iteritems():
pathway.setOmicAdjustedSignificanceValues(omic, {adjust_method: pvalues[pathway_id] for adjust_method, pvalues in adjusted_pvalues[omic].iteritems()})
for method, combined_pvalue in pathway.getCombinedSignificancePvalues().iteritems():
pathway.setMethodAdjustedCombinedSignificanceValues(method, {adjust_method: combined_pvalues[pathway_id] for adjust_method, combined_pvalues in adjusted_combined_pvalues[method].iteritems()})
logging.info("SUMMARY: " + str(totalMatchedKeggPathways) + " Matched Pathways of " + str(totalKeggPathways) + "in KEGG; Total input Genes = " + str(totalInputMatchedGenes) + "; SUMMARY: Total input Compounds = " + str(totalInputMatchedCompounds))
for key in totalFeaturesByOmic:
logging.info("SUMMARY: Total " + key + " Features = " + str(totalFeaturesByOmic.get(key)))
logging.info("SUMMARY: Total " + key + " Relevant Features = " + str(totalRelevantFeaturesByOmic.get(key)))
self.summary= [totalKeggPathways, totalMatchedKeggPathways, totalInputMatchedGenes, totalInputMatchedCompounds, totalFeaturesByOmic, totalRelevantFeaturesByOmic]
#TODO: REVIEW THE SUMMARY GENERATION
return self.summary
pw.start()
pr.start()
pw.join()
pr.terminate()
print '---' * 20
num = Value('d', 0.0)
arr = Array('i', range(10))
p = Process(target=f, args=(num, arr))
p.start()
p.join()
print num.value
print arr[:]
print '--' * 20
manager = Manager()
d = manager.dict()
ll = manager.list(range(10))
p = Process(target=manage_f, args=(d, ll))
p2 = Process(target=manage_f2, args=(d, ll))
p.start()
p2.start()
p.join()
print d
print ll
help="1 for relative 0 for absolute")
args = vars(ap.parse_args())
initializeUser(args['user'])
global calibration
calibrationIndexes = [
'position', 'gain', 'threshL0', 'threshL1', 'getBg', 'threshH', 'yth',
'plane', 'depth', 'final'
]
calibration = {}
for i in xrange(len(calibrationIndexes)):
calibration[calibrationIndexes[i]] = i
# variables for multi-processing
manager = Manager()
key = manager.dict() # get keyboard events
kill = manager.dict() # killing other processes or not
key['status'] = False
key['info'] = ""
kill['kill'] = False
if args['step'] == 'final':
p = Process(target=keylogger.log, args=(key, kill))
p.start()
global timers
timers = [Timer() for i in xrange(4)]
mainLoop(calibration[args['step']], kill, key, args['user'], args['mode'])
def run(self):
if self._mode == 'batch':
manager = Manager()
self._logger_state = manager.dict()
self._node_states = manager.dict()
self._node_state_update_timestamps = manager.dict()
self._context = zmq.Context()
self._session_id = np.random.randint(10000, 11000)
nodes = []
node_clients = []
self._node_token = NodeToken(0, self._context, 'tcp://127.0.0.1', 18534, 28534)
for i in range(1, self._num_threads+1):
token = NodeToken(i, self._context, 'tcp://127.0.0.1', 18534 + i, 28534)
node = NodeClient(token)
node_clients.append(node)
self._node_states[token.node_id] = rc.STATUS_CODE_NOT_READY
self._node_state_update_timestamps[token.node_id] = datetime.now()
rct = ReCoDeNode(token, self._calibration_frame, self._init_params, self._input_params)
p = Process(target=rct.run,
args=(self._session_id, token,
self._node_states, self._node_state_update_timestamps, self._logger_state))
nodes.append(p)
p.start()
time.sleep(0.1)
# start logger
token = NodeToken(-1, self._context, 'tcp://127.0.0.1', -1, 28534)
logger = Logger(token)
self._logger_state = {0: rc.STATUS_CODE_NOT_READY}
logger_process = Process(target=logger.start, args=(self._session_id, self._logger_state))
logger_process.start()
time.sleep(0.1)
# connect to logger
self._pub_socket = self._context.socket(zmq.PUSH)
self._pub_socket.connect(self._node_token.ip_address + ":" + str(self._node_token.publishing_port))
self._log(rc.MESSAGE_TYPE_INFO_RESPONSE, 'Welcome to this session')
# connect to RC nodes
for i in range(self._num_threads):
node_clients[i].connect()
self._broadcast(node_clients, self._session_id, 1, 'start')
self._broadcast(node_clients, self._session_id, 2, 'process_file')
# close RC nodes
self._broadcast(node_clients, self._session_id, 3, 'close')
for i in range(self._num_threads):
node_clients[i].close()
nodes[i].join()
# close logger
self._log(rc.MESSAGE_TYPE_INFO_RESPONSE, 'close')
logger_process.join()
def outlier_detection(data, data_type, parameters):
"""
Performs outlier detection on data of given data_type according to specified parameters.
Input:
data (dict): input data.
example: data = {1: {'segments': record 1 ECG segments, 'R': record 1 r peaks},
2: {'segments': record 2 ECG segments, 'R': record 2 r peaks},
...}
data_type (string): data type to be analyzed
parameters (dict): filter parameters.
Example: parameters = {'method': 'dbscan', ...}
Output:
output (dict): output data where keys correspond to record id numbers.
example: output = { 1: {-1: record 1 outlier indexes, '0': record 1 cluster 0 indexes},
2: {-1: record 2 outlier indexes, '0': record 2 cluster 0 indexes},
...}
Configurable fields:{"name": "??.??", "config": {}, "inputs": ["data", "data_type", "parameters"], "outputs": ["output"]}
See Also:
Notes:
Example:
"""
if parameters['method'] == 'dbscan':
method = outlier.outliers_dbscan
else:
raise TypeError, "Method %s not implemented." % parameters['method']
# create work queue
work_queue = Queue()
manager = Manager()
output = manager.dict()
output['info'] = parameters
parameters.pop('method')
# fill queue
for recid in data.keys():
if recid == 'info': continue
work_queue.put({
'function': method,
'data': data.get(recid).get('segments'),
'parameters': parameters,
'recid': recid
})
# create N processes and associate them with the work_queue and do_work function
processes = [
Process(target=do_work, args=(
work_queue,
output,
)) for _ in range(NUMBER_PROCESSES)
]
# lauch processes
for p in processes:
p.start()
# wait for processes to finish
print "waiting ..."
for p in processes:
p.join()
print "wait is over ..."
for p in processes:
p.terminate()
return output
trials2 = []
for folder in os.listdir("smooths/second/"):
for filename in os.listdir("smooths/second/" + folder):
trials2.append("smooths/second/" + folder + "/" + filename)
trials3 = []
for folder in os.listdir("smooths/third/"):
for filename in os.listdir("smooths/third/" + folder):
trials3.append("smooths/third/" + folder + "/" + filename)
corpus = []
with open(processing.dataset_folder + "test_set.txt") as file:
for l in file.readlines():
corpus.append(l.strip())
manager = Manager()
returns = manager.dict()
for id in tqdm(range(0, len(trials1), 5)):
results = []
ids = id
p1 = Process(target=processing.compute_tagging,
args=(corpus, first_lex_in, first_lex_out, second_lex_in,
second_lex_out, automaton1, automaton2,
class_cluster, trials1[ids], trials2[ids], returns,
trials1[ids]))
p1.start()
ids += 1
p2 = Process(target=processing.compute_tagging,
args=(corpus, first_lex_in, first_lex_out, second_lex_in,
second_lex_out, automaton1, automaton2,
class_cluster, trials1[ids], trials2[ids], returns,
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
__author__ = 'but0n'
from multiprocessing import Pool, Manager
from bs4 import BeautifulSoup
import time, random, requests, sqlite3, os
server = Manager()
host = 'http://www.80s.tw'
screen = server.dict({'label' : 'NONE', 'url' : 'http://baidu.com', 'title':'none', 'IMG':'none', 'detail':'none', 'link':'none', 'index':0, 'total':10})
def mLog(opt):
os.system('clear')
print('\033[41;30m MESSAGE: %s\033[m' % opt['label'])
print('\033[46;30m PATH: %10s\033[m\n' % opt['url'])
print('\033[0;35m TITLE\033[m:\t%s' % opt['title'])
print('\033[0;35m IMG\033[m:\t%s' % opt['IMG'][:30]+'...')
print('\033[0;34m DETAIL\033[m:%s' % opt['detail'][:60]+'...')
print('\033[0;36m LINK\033[m:\t%s' % opt['link'][:60]+'...')
bar_status = opt['index']*40/opt['total']
status = opt['index']*100/opt['total']
print('\n[%-40s]%s(%d/%d)' % ('>'*bar_status, str(status)+'%', opt['index'], opt['total']))
class domPa(object):
def __init__(self, path, section = 'a', title = '.title', img = '.img', detail = '.detail'):
self.path = path
self.page = requests.get(host+path)
# bar
def bar(name, data):
print(name)
for i in range(3):
data[i] = i
print("Tick")
time.sleep(1)
print(data)
if __name__ == '__main__':
# Start bar as a process
manager = Manager()
data = manager.dict()
p = multiprocessing.Process(target=bar, args=["test", data])
p.start()
# Wait for 10 seconds or until process finishes
p.join(5)
# If thread is still active
if p.is_alive():
print("running... let's kill it...")
# Terminate
p.terminate()
p.join()
print(data)
class SQLDB:
PRAGMAS = """
pragma journal_mode=WAL;
"""
CREATE_CLAIM_TABLE = """
create table if not exists claim (
claim_hash bytes primary key,
claim_id text not null,
claim_name text not null,
normalized text not null,
txo_hash bytes not null,
tx_position integer not null,
amount integer not null,
timestamp integer not null, -- last updated timestamp
creation_timestamp integer not null,
height integer not null, -- last updated height
creation_height integer not null,
activation_height integer,
expiration_height integer not null,
release_time integer not null,
short_url text not null, -- normalized#shortest-unique-claim_id
canonical_url text, -- channel's-short_url/normalized#shortest-unique-claim_id-within-channel
title text,
author text,
description text,
claim_type integer,
reposted integer default 0,
-- streams
stream_type text,
media_type text,
fee_amount integer default 0,
fee_currency text,
duration integer,
-- reposts
reposted_claim_hash bytes,
-- claims which are channels
public_key_bytes bytes,
public_key_hash bytes,
claims_in_channel integer,
-- claims which are inside channels
channel_hash bytes,
channel_join integer, -- height at which claim got valid signature / joined channel
signature bytes,
signature_digest bytes,
signature_valid bool,
effective_amount integer not null default 0,
support_amount integer not null default 0,
trending_group integer not null default 0,
trending_mixed integer not null default 0,
trending_local integer not null default 0,
trending_global integer not null default 0
);
create index if not exists claim_normalized_idx on claim (normalized, activation_height);
create index if not exists claim_channel_hash_idx on claim (channel_hash, signature, claim_hash);
create index if not exists claim_claims_in_channel_idx on claim (signature_valid, channel_hash, normalized);
create index if not exists claim_txo_hash_idx on claim (txo_hash);
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
create index if not exists claim_expiration_height_idx on claim (expiration_height);
"""
CREATE_SUPPORT_TABLE = """
create table if not exists support (
txo_hash bytes primary key,
tx_position integer not null,
height integer not null,
claim_hash bytes not null,
amount integer not null
);
create index if not exists support_claim_hash_idx on support (claim_hash, height);
"""
CREATE_TAG_TABLE = """
create table if not exists tag (
tag text not null,
claim_hash bytes not null,
height integer not null
);
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
"""
CREATE_CLAIMTRIE_TABLE = """
create table if not exists claimtrie (
normalized text primary key,
claim_hash bytes not null,
last_take_over_height integer not null
);
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
"""
SEARCH_INDEXES = """
-- used by any tag clouds
create index if not exists tag_tag_idx on tag (tag, claim_hash);
-- common ORDER BY
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
create unique index if not exists claim_trending_group_mixed_idx on claim (trending_group, trending_mixed, claim_hash);
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_global, trending_mixed, claim_hash);
create unique index if not exists claim_type_release_idx on claim (claim_type, release_time, claim_hash);
create unique index if not exists claim_type_effective_amount_idx on claim (claim_type, effective_amount, claim_hash);
create unique index if not exists channel_hash_release_time_idx on claim (channel_hash, release_time, claim_hash);
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, trending_global, trending_mixed, claim_hash);
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, effective_amount, claim_hash);
create unique index if not exists filter_stream_duration_idx on claim (duration, trending_global, trending_mixed, claim_hash);
-- TODO: verify that all indexes below are used
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
"""
TAG_INDEXES = '\n'.join(
f"create unique index if not exists tag_{tag_key}_idx on tag (tag, claim_hash) WHERE tag='{tag_value}';"
for tag_value, tag_key in COMMON_TAGS.items())
CREATE_TABLES_QUERY = (CREATE_CLAIM_TABLE + CREATE_FULL_TEXT_SEARCH +
CREATE_SUPPORT_TABLE + CREATE_CLAIMTRIE_TABLE +
CREATE_TAG_TABLE)
def __init__(self, main, path: str, blocking_channels: list,
filtering_channels: list, trending: list):
self.main = main
self._db_path = path
self.db = None
self.logger = class_logger(__name__, self.__class__.__name__)
self.ledger = Ledger if main.coin.NET == 'mainnet' else RegTestLedger
self._fts_synced = False
self.state_manager = None
self.blocked_streams = None
self.blocked_channels = None
self.blocking_channel_hashes = {
unhexlify(channel_id)[::-1]
for channel_id in blocking_channels if channel_id
}
self.filtered_streams = None
self.filtered_channels = None
self.filtering_channel_hashes = {
unhexlify(channel_id)[::-1]
for channel_id in filtering_channels if channel_id
}
self.trending = trending
def open(self):
self.db = apsw.Connection(
self._db_path,
flags=(apsw.SQLITE_OPEN_READWRITE | apsw.SQLITE_OPEN_CREATE
| apsw.SQLITE_OPEN_URI))
def exec_factory(cursor, statement, bindings):
tpl = namedtuple('row', (d[0] for d in cursor.getdescription()))
cursor.setrowtrace(lambda cursor, row: tpl(*row))
return True
self.db.setexectrace(exec_factory)
self.execute(self.PRAGMAS)
self.execute(self.CREATE_TABLES_QUERY)
register_canonical_functions(self.db)
self.state_manager = Manager()
self.blocked_streams = self.state_manager.dict()
self.blocked_channels = self.state_manager.dict()
self.filtered_streams = self.state_manager.dict()
self.filtered_channels = self.state_manager.dict()
self.update_blocked_and_filtered_claims()
for algorithm in self.trending:
algorithm.install(self.db)
def close(self):
if self.db is not None:
self.db.close()
if self.state_manager is not None:
self.state_manager.shutdown()
def update_blocked_and_filtered_claims(self):
self.update_claims_from_channel_hashes(self.blocked_streams,
self.blocked_channels,
self.blocking_channel_hashes)
self.update_claims_from_channel_hashes(self.filtered_streams,
self.filtered_channels,
self.filtering_channel_hashes)
self.filtered_streams.update(self.blocked_streams)
self.filtered_channels.update(self.blocked_channels)
def update_claims_from_channel_hashes(self, shared_streams,
shared_channels, channel_hashes):
streams, channels = {}, {}
if channel_hashes:
sql = query(
"SELECT claim.channel_hash, claim.reposted_claim_hash, reposted.claim_type "
"FROM claim JOIN claim AS reposted ON (reposted.claim_hash=claim.reposted_claim_hash)",
**{
'claim.reposted_claim_hash__is_not_null': 1,
'claim.channel_hash__in': channel_hashes
})
for blocked_claim in self.execute(*sql):
if blocked_claim.claim_type == CLAIM_TYPES['stream']:
streams[blocked_claim.
reposted_claim_hash] = blocked_claim.channel_hash
elif blocked_claim.claim_type == CLAIM_TYPES['channel']:
channels[blocked_claim.
reposted_claim_hash] = blocked_claim.channel_hash
shared_streams.clear()
shared_streams.update(streams)
shared_channels.clear()
shared_channels.update(channels)
@staticmethod
def _insert_sql(table: str, data: dict) -> Tuple[str, list]:
columns, values = [], []
for column, value in data.items():
columns.append(column)
values.append(value)
sql = (f"INSERT INTO {table} ({', '.join(columns)}) "
f"VALUES ({', '.join(['?'] * len(values))})")
return sql, values
@staticmethod
def _update_sql(table: str, data: dict, where: str,
constraints: Union[list, tuple]) -> Tuple[str, list]:
columns, values = [], []
for column, value in data.items():
columns.append(f"{column} = ?")
values.append(value)
values.extend(constraints)
return f"UPDATE {table} SET {', '.join(columns)} WHERE {where}", values
@staticmethod
def _delete_sql(table: str, constraints: dict) -> Tuple[str, dict]:
where, values = constraints_to_sql(constraints)
return f"DELETE FROM {table} WHERE {where}", values
def execute(self, *args):
return self.db.cursor().execute(*args)
def executemany(self, *args):
return self.db.cursor().executemany(*args)
def begin(self):
self.execute('begin;')
def commit(self):
self.execute('commit;')
def _upsertable_claims(self,
txos: List[Output],
header,
clear_first=False):
claim_hashes, claims, tags = set(), [], {}
for txo in txos:
tx = txo.tx_ref.tx
try:
assert txo.claim_name
assert txo.normalized_name
except:
#self.logger.exception(f"Could not decode claim name for {tx.id}:{txo.position}.")
continue
claim_hash = txo.claim_hash
claim_hashes.add(claim_hash)
claim_record = {
'claim_hash': claim_hash,
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'normalized': txo.normalized_name,
'txo_hash': txo.ref.hash,
'tx_position': tx.position,
'amount': txo.amount,
'timestamp': header['timestamp'],
'height': tx.height,
'title': None,
'description': None,
'author': None,
'duration': None,
'claim_type': None,
'stream_type': None,
'media_type': None,
'release_time': None,
'fee_currency': None,
'fee_amount': 0,
'reposted_claim_hash': None
}
claims.append(claim_record)
try:
claim = txo.claim
except:
#self.logger.exception(f"Could not parse claim protobuf for {tx.id}:{txo.position}.")
continue
if claim.is_stream:
claim_record['claim_type'] = CLAIM_TYPES['stream']
claim_record['media_type'] = claim.stream.source.media_type
claim_record['stream_type'] = STREAM_TYPES[guess_stream_type(
claim_record['media_type'])]
claim_record['title'] = claim.stream.title
claim_record['description'] = claim.stream.description
claim_record['author'] = claim.stream.author
if claim.stream.video and claim.stream.video.duration:
claim_record['duration'] = claim.stream.video.duration
if claim.stream.audio and claim.stream.audio.duration:
claim_record['duration'] = claim.stream.audio.duration
if claim.stream.release_time:
claim_record['release_time'] = claim.stream.release_time
if claim.stream.has_fee:
fee = claim.stream.fee
if isinstance(fee.currency, str):
claim_record['fee_currency'] = fee.currency.lower()
if isinstance(fee.amount, Decimal):
claim_record['fee_amount'] = int(fee.amount * 1000)
elif claim.is_repost:
claim_record[
'reposted_claim_hash'] = claim.repost.reference.claim_hash
elif claim.is_channel:
claim_record['claim_type'] = CLAIM_TYPES['channel']
for tag in clean_tags(claim.message.tags):
tags[(tag, claim_hash)] = (tag, claim_hash, tx.height)
if clear_first:
self._clear_claim_metadata(claim_hashes)
if tags:
self.executemany(
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)",
tags.values())
return claims
def insert_claims(self, txos: List[Output], header):
claims = self._upsertable_claims(txos, header)
if claims:
self.executemany(
"""
INSERT OR IGNORE INTO claim (
claim_hash, claim_id, claim_name, normalized, txo_hash, tx_position, amount,
claim_type, media_type, stream_type, timestamp, creation_timestamp,
fee_currency, fee_amount, title, description, author, duration, height, reposted_claim_hash,
creation_height, release_time, activation_height, expiration_height, short_url)
VALUES (
:claim_hash, :claim_id, :claim_name, :normalized, :txo_hash, :tx_position, :amount,
:claim_type, :media_type, :stream_type, :timestamp, :timestamp,
:fee_currency, :fee_amount, :title, :description, :author, :duration, :height, :reposted_claim_hash, :height,
CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE :timestamp END,
CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END,
CASE WHEN :height >= 137181 THEN :height+2102400 ELSE :height+262974 END,
:claim_name||COALESCE(
(SELECT shortest_id(claim_id, :claim_id) FROM claim WHERE normalized = :normalized),
'#'||substr(:claim_id, 1, 1)
)
)""", claims)
def update_claims(self, txos: List[Output], header):
claims = self._upsertable_claims(txos, header, clear_first=True)
if claims:
self.executemany(
"""
UPDATE claim SET
txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height,
claim_type=:claim_type, media_type=:media_type, stream_type=:stream_type,
timestamp=:timestamp, fee_amount=:fee_amount, fee_currency=:fee_currency,
title=:title, duration=:duration, description=:description, author=:author, reposted_claim_hash=:reposted_claim_hash,
release_time=CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE release_time END
WHERE claim_hash=:claim_hash;
""", claims)
def delete_claims(self, claim_hashes: Set[bytes]):
""" Deletes claim supports and from claimtrie in case of an abandon. """
if claim_hashes:
affected_channels = self.execute(
*query("SELECT channel_hash FROM claim",
channel_hash__is_not_null=1,
claim_hash__in=claim_hashes)).fetchall()
for table in ('claim', 'support', 'claimtrie'):
self.execute(
*self._delete_sql(table, {'claim_hash__in': claim_hashes}))
self._clear_claim_metadata(claim_hashes)
return {r.channel_hash for r in affected_channels}
return set()
def _clear_claim_metadata(self, claim_hashes: Set[bytes]):
if claim_hashes:
for table in ('tag', ): # 'language', 'location', etc
self.execute(
*self._delete_sql(table, {'claim_hash__in': claim_hashes}))
def split_inputs_into_claims_supports_and_other(self, txis):
txo_hashes = {txi.txo_ref.hash for txi in txis}
claims = self.execute(
*query("SELECT txo_hash, claim_hash, normalized FROM claim",
txo_hash__in=txo_hashes)).fetchall()
txo_hashes -= {r.txo_hash for r in claims}
supports = {}
if txo_hashes:
supports = self.execute(
*query("SELECT txo_hash, claim_hash FROM support",
txo_hash__in=txo_hashes)).fetchall()
txo_hashes -= {r.txo_hash for r in supports}
return claims, supports, txo_hashes
def insert_supports(self, txos: List[Output]):
supports = []
for txo in txos:
tx = txo.tx_ref.tx
supports.append((txo.ref.hash, tx.position, tx.height,
txo.claim_hash, txo.amount))
if supports:
self.executemany(
"INSERT OR IGNORE INTO support ("
" txo_hash, tx_position, height, claim_hash, amount"
") "
"VALUES (?, ?, ?, ?, ?)", supports)
def delete_supports(self, txo_hashes: Set[bytes]):
if txo_hashes:
self.execute(
*self._delete_sql('support', {'txo_hash__in': txo_hashes}))
def calculate_reposts(self, txos: List[Output]):
targets = set()
for txo in txos:
try:
claim = txo.claim
except:
continue
if claim.is_repost:
targets.add((claim.repost.reference.claim_hash, ))
if targets:
self.executemany(
"""
UPDATE claim SET reposted = (
SELECT count(*) FROM claim AS repost WHERE repost.reposted_claim_hash = claim.claim_hash
)
WHERE claim_hash = ?
""", targets)
def validate_channel_signatures(self, height, new_claims, updated_claims,
spent_claims, affected_channels, timer):
if not new_claims and not updated_claims and not spent_claims:
return
sub_timer = timer.add_timer('segregate channels and signables')
sub_timer.start()
channels, new_channel_keys, signables = {}, {}, {}
for txo in chain(new_claims, updated_claims):
try:
claim = txo.claim
except:
continue
if claim.is_channel:
channels[txo.claim_hash] = txo
new_channel_keys[
txo.claim_hash] = claim.channel.public_key_bytes
else:
signables[txo.claim_hash] = txo
sub_timer.stop()
sub_timer = timer.add_timer('make list of channels we need to lookup')
sub_timer.start()
missing_channel_keys = set()
for txo in signables.values():
claim = txo.claim
if claim.is_signed and claim.signing_channel_hash not in new_channel_keys:
missing_channel_keys.add(claim.signing_channel_hash)
sub_timer.stop()
sub_timer = timer.add_timer('lookup missing channels')
sub_timer.start()
all_channel_keys = {}
if new_channel_keys or missing_channel_keys or affected_channels:
all_channel_keys = dict(
self.execute(
*query("SELECT claim_hash, public_key_bytes FROM claim",
claim_hash__in=set(new_channel_keys)
| missing_channel_keys | affected_channels)))
sub_timer.stop()
sub_timer = timer.add_timer('prepare for updating claims')
sub_timer.start()
changed_channel_keys = {}
for claim_hash, new_key in new_channel_keys.items():
if claim_hash not in all_channel_keys or all_channel_keys[
claim_hash] != new_key:
all_channel_keys[claim_hash] = new_key
changed_channel_keys[claim_hash] = new_key
claim_updates = []
for claim_hash, txo in signables.items():
claim = txo.claim
update = {
'claim_hash': claim_hash,
'channel_hash': None,
'signature': None,
'signature_digest': None,
'signature_valid': None
}
if claim.is_signed:
update.update({
'channel_hash':
claim.signing_channel_hash,
'signature':
txo.get_encoded_signature(),
'signature_digest':
txo.get_signature_digest(self.ledger),
'signature_valid':
0
})
claim_updates.append(update)
sub_timer.stop()
sub_timer = timer.add_timer(
'find claims affected by a change in channel key')
sub_timer.start()
if changed_channel_keys:
sql = f"""
SELECT * FROM claim WHERE
channel_hash IN ({','.join('?' for _ in changed_channel_keys)}) AND
signature IS NOT NULL
"""
for affected_claim in self.execute(sql,
changed_channel_keys.keys()):
if affected_claim.claim_hash not in signables:
claim_updates.append({
'claim_hash': affected_claim.claim_hash,
'channel_hash': affected_claim.channel_hash,
'signature': affected_claim.signature,
'signature_digest': affected_claim.signature_digest,
'signature_valid': 0
})
sub_timer.stop()
sub_timer = timer.add_timer('verify signatures')
sub_timer.start()
for update in claim_updates:
channel_pub_key = all_channel_keys.get(update['channel_hash'])
if channel_pub_key and update['signature']:
update['signature_valid'] = Output.is_signature_valid(
bytes(update['signature']),
bytes(update['signature_digest']), channel_pub_key)
sub_timer.stop()
sub_timer = timer.add_timer('update claims')
sub_timer.start()
if claim_updates:
self.executemany(
f"""
UPDATE claim SET
channel_hash=:channel_hash, signature=:signature, signature_digest=:signature_digest,
signature_valid=:signature_valid,
channel_join=CASE
WHEN signature_valid=1 AND :signature_valid=1 AND channel_hash=:channel_hash THEN channel_join
WHEN :signature_valid=1 THEN {height}
END,
canonical_url=CASE
WHEN signature_valid=1 AND :signature_valid=1 AND channel_hash=:channel_hash THEN canonical_url
WHEN :signature_valid=1 THEN
(SELECT short_url FROM claim WHERE claim_hash=:channel_hash)||'/'||
claim_name||COALESCE(
(SELECT shortest_id(other_claim.claim_id, claim.claim_id) FROM claim AS other_claim
WHERE other_claim.signature_valid = 1 AND
other_claim.channel_hash = :channel_hash AND
other_claim.normalized = claim.normalized),
'#'||substr(claim_id, 1, 1)
)
END
WHERE claim_hash=:claim_hash;
""", claim_updates)
sub_timer.stop()
sub_timer = timer.add_timer('update claims affected by spent channels')
sub_timer.start()
if spent_claims:
self.execute(
f"""
UPDATE claim SET
signature_valid=CASE WHEN signature IS NOT NULL THEN 0 END,
channel_join=NULL, canonical_url=NULL
WHERE channel_hash IN ({','.join('?' for _ in spent_claims)})
""", spent_claims)
sub_timer.stop()
sub_timer = timer.add_timer('update channels')
sub_timer.start()
if channels:
self.executemany(
"""
UPDATE claim SET
public_key_bytes=:public_key_bytes,
public_key_hash=:public_key_hash
WHERE claim_hash=:claim_hash""", [{
'claim_hash':
claim_hash,
'public_key_bytes':
txo.claim.channel.public_key_bytes,
'public_key_hash':
self.ledger.address_to_hash160(
self.ledger.public_key_to_address(
txo.claim.channel.public_key_bytes))
} for claim_hash, txo in channels.items()])
sub_timer.stop()
sub_timer = timer.add_timer('update claims_in_channel counts')
sub_timer.start()
if all_channel_keys:
self.executemany(
f"""
UPDATE claim SET
claims_in_channel=(
SELECT COUNT(*) FROM claim AS claim_in_channel
WHERE claim_in_channel.signature_valid=1 AND
claim_in_channel.channel_hash=claim.claim_hash
)
WHERE claim_hash = ?
""",
[(channel_hash, ) for channel_hash in all_channel_keys.keys()])
sub_timer.stop()
sub_timer = timer.add_timer('update blocked claims list')
sub_timer.start()
if (self.blocking_channel_hashes.intersection(all_channel_keys) or
self.filtering_channel_hashes.intersection(all_channel_keys)):
self.update_blocked_and_filtered_claims()
sub_timer.stop()
def _update_support_amount(self, claim_hashes):
if claim_hashes:
self.execute(
f"""
UPDATE claim SET
support_amount = COALESCE(
(SELECT SUM(amount) FROM support WHERE support.claim_hash=claim.claim_hash), 0
)
WHERE claim_hash IN ({','.join('?' for _ in claim_hashes)})
""", claim_hashes)
def _update_effective_amount(self, height, claim_hashes=None):
self.execute(
f"UPDATE claim SET effective_amount = amount + support_amount "
f"WHERE activation_height = {height}")
if claim_hashes:
self.execute(
f"UPDATE claim SET effective_amount = amount + support_amount "
f"WHERE activation_height < {height} "
f" AND claim_hash IN ({','.join('?' for _ in claim_hashes)})",
claim_hashes)
def _calculate_activation_height(self, height):
last_take_over_height = f"""COALESCE(
(SELECT last_take_over_height FROM claimtrie
WHERE claimtrie.normalized=claim.normalized),
{height}
)
"""
self.execute(f"""
UPDATE claim SET activation_height =
{height} + min(4032, cast(({height} - {last_take_over_height}) / 32 AS INT))
WHERE activation_height IS NULL
""")
def _perform_overtake(self, height, changed_claim_hashes, deleted_names):
deleted_names_sql = claim_hashes_sql = ""
if changed_claim_hashes:
claim_hashes_sql = f"OR claim_hash IN ({','.join('?' for _ in changed_claim_hashes)})"
if deleted_names:
deleted_names_sql = f"OR normalized IN ({','.join('?' for _ in deleted_names)})"
overtakes = self.execute(
f"""
SELECT winner.normalized, winner.claim_hash,
claimtrie.claim_hash AS current_winner,
MAX(winner.effective_amount) AS max_winner_effective_amount
FROM (
SELECT normalized, claim_hash, effective_amount FROM claim
WHERE normalized IN (
SELECT normalized FROM claim WHERE activation_height={height} {claim_hashes_sql}
) {deleted_names_sql}
ORDER BY effective_amount DESC, height ASC, tx_position ASC
) AS winner LEFT JOIN claimtrie USING (normalized)
GROUP BY winner.normalized
HAVING current_winner IS NULL OR current_winner <> winner.claim_hash
""",
list(changed_claim_hashes) + deleted_names)
for overtake in overtakes:
if overtake.current_winner:
self.execute(
f"UPDATE claimtrie SET claim_hash = ?, last_take_over_height = {height} "
f"WHERE normalized = ?",
(overtake.claim_hash, overtake.normalized))
else:
self.execute(
f"INSERT INTO claimtrie (claim_hash, normalized, last_take_over_height) "
f"VALUES (?, ?, {height})",
(overtake.claim_hash, overtake.normalized))
self.execute(
f"UPDATE claim SET activation_height = {height} WHERE normalized = ? "
f"AND (activation_height IS NULL OR activation_height > {height})",
(overtake.normalized, ))
def _copy(self, height):
if height > 50:
self.execute(f"DROP TABLE claimtrie{height-50}")
self.execute(
f"CREATE TABLE claimtrie{height} AS SELECT * FROM claimtrie")
def update_claimtrie(self, height, changed_claim_hashes, deleted_names,
timer):
r = timer.run
r(self._calculate_activation_height, height)
r(self._update_support_amount, changed_claim_hashes)
r(self._update_effective_amount, height, changed_claim_hashes)
r(self._perform_overtake, height, changed_claim_hashes,
list(deleted_names))
r(self._update_effective_amount, height)
r(self._perform_overtake, height, [], [])
def get_expiring(self, height):
return self.execute(
f"SELECT claim_hash, normalized FROM claim WHERE expiration_height = {height}"
)
def advance_txs(self, height, all_txs, header, daemon_height, timer):
insert_claims = []
update_claims = []
update_claim_hashes = set()
delete_claim_hashes = set()
insert_supports = []
delete_support_txo_hashes = set()
recalculate_claim_hashes = set(
) # added/deleted supports, added/updated claim
deleted_claim_names = set()
delete_others = set()
body_timer = timer.add_timer('body')
for position, (etx, txid) in enumerate(all_txs):
tx = timer.run(Transaction,
etx.raw,
height=height,
position=position)
# Inputs
spent_claims, spent_supports, spent_others = timer.run(
self.split_inputs_into_claims_supports_and_other, tx.inputs)
body_timer.start()
delete_claim_hashes.update({r.claim_hash for r in spent_claims})
deleted_claim_names.update({r.normalized for r in spent_claims})
delete_support_txo_hashes.update(
{r.txo_hash
for r in spent_supports})
recalculate_claim_hashes.update(
{r.claim_hash
for r in spent_supports})
delete_others.update(spent_others)
# Outputs
for output in tx.outputs:
if output.is_support:
insert_supports.append(output)
recalculate_claim_hashes.add(output.claim_hash)
elif output.script.is_claim_name:
insert_claims.append(output)
recalculate_claim_hashes.add(output.claim_hash)
elif output.script.is_update_claim:
claim_hash = output.claim_hash
update_claims.append(output)
recalculate_claim_hashes.add(claim_hash)
body_timer.stop()
skip_update_claim_timer = timer.add_timer(
'skip update of abandoned claims')
skip_update_claim_timer.start()
for updated_claim in list(update_claims):
if updated_claim.ref.hash in delete_others:
update_claims.remove(updated_claim)
for updated_claim in update_claims:
claim_hash = updated_claim.claim_hash
delete_claim_hashes.discard(claim_hash)
update_claim_hashes.add(claim_hash)
skip_update_claim_timer.stop()
skip_insert_claim_timer = timer.add_timer(
'skip insertion of abandoned claims')
skip_insert_claim_timer.start()
for new_claim in list(insert_claims):
if new_claim.ref.hash in delete_others:
if new_claim.claim_hash not in update_claim_hashes:
insert_claims.remove(new_claim)
skip_insert_claim_timer.stop()
skip_insert_support_timer = timer.add_timer(
'skip insertion of abandoned supports')
skip_insert_support_timer.start()
for new_support in list(insert_supports):
if new_support.ref.hash in delete_others:
insert_supports.remove(new_support)
skip_insert_support_timer.stop()
expire_timer = timer.add_timer('recording expired claims')
expire_timer.start()
for expired in self.get_expiring(height):
delete_claim_hashes.add(expired.claim_hash)
deleted_claim_names.add(expired.normalized)
expire_timer.stop()
r = timer.run
r(update_full_text_search, 'before-delete', delete_claim_hashes,
self.db.cursor(), self.main.first_sync)
affected_channels = r(self.delete_claims, delete_claim_hashes)
r(self.delete_supports, delete_support_txo_hashes)
r(self.insert_claims, insert_claims, header)
r(self.calculate_reposts, insert_claims)
r(update_full_text_search,
'after-insert', [txo.claim_hash for txo in insert_claims],
self.db.cursor(), self.main.first_sync)
r(update_full_text_search,
'before-update', [txo.claim_hash for txo in update_claims],
self.db.cursor(), self.main.first_sync)
r(self.update_claims, update_claims, header)
r(update_full_text_search,
'after-update', [txo.claim_hash for txo in update_claims],
self.db.cursor(), self.main.first_sync)
r(self.validate_channel_signatures,
height,
insert_claims,
update_claims,
delete_claim_hashes,
affected_channels,
forward_timer=True)
r(self.insert_supports, insert_supports)
r(self.update_claimtrie,
height,
recalculate_claim_hashes,
deleted_claim_names,
forward_timer=True)
for algorithm in self.trending:
r(algorithm.run, self.db.cursor(), height, daemon_height,
recalculate_claim_hashes)
if not self._fts_synced and self.main.first_sync and height == daemon_height:
r(first_sync_finished, self.db.cursor())
self._fts_synced = True
class SIB():
"""Share Is Beautiful
"""
def __init__(self):
logging.basicConfig(
filename='SIB.log',
format=
'%(asctime)s | %(process)d | %(processName)s | %(levelname)s | %(module)s | %(funcName)s | %(message)s',
level=logging.DEBUG)
#shared objects for inter-process communication
#TODO: WTH is there so much shared state?
self.raw_in_queue = Queue()
self.json_call_queue = Queue()
self.json_out_queue = Queue()
self.manager = Manager()
self.json_response_dict = self.manager.dict()
#self.listen_sockets = self.manager.list()
self.new_sockets = Queue()
self.congest_sent_queue = Queue()
self.congest_recv_queue = Queue()
self.congest_connect_state = self.manager.dict()
self.ss = SocketServer()
#self.ss.listen_sockets = self.listen_sockets
self.ss.new_sockets = self.new_sockets
self.ss.raw_in_queue = self.raw_in_queue
self.pp = PacketPrePostprocessor()
self.pp.raw_in_queue = self.raw_in_queue
self.pp.json_queue = self.json_call_queue
self.pp.json_response_dict = self.json_response_dict
self.pp.json_out_queue = self.json_out_queue
self.pp.congest_sent_queue = self.congest_sent_queue
self.pp.congest_recv_queue = self.congest_recv_queue
self.js = JSONServer()
self.js.json_call_queue = self.json_call_queue
self.js.json_response_dict = self.json_response_dict
self.js.json_out_queue = self.json_out_queue
self.js.congest_connect_state = self.congest_connect_state
self.congest = CongestionManager()
self.congest.shared_sent_queue = self.congest_sent_queue
self.congest.shared_recv_queue = self.congest_recv_queue
self.congest.connect_summary = self.congest_connect_state
def display(self):
"""Prints contents of SIB for debug purposes.
"""
#self.raw_in_queue = Queue()
#self.json_call_queue = Queue()
#self.json_out_queue = Queue()
#self.json_response_dict = self.manager.dict()
print '=========Printing details of SIB object========='
print "raw_in_queue size: %s" % (self.raw_in_queue.qsize())
print "json_call_queue size: %s" % (self.json_call_queue.qsize())
print "json_out_queuesize: %s" % (self.json_out_queue.qsize())
print "json_response_dict size: %s" % (len(self.json_response_dict))
print '------------------------------------------------------'
def run_once(self):
self.ss.serve_once()
self.pp.run_pre_once()
self.js.serve_once()
self.pp.run_post_once()
def run_forever(self):
logging.info('serving forever')
self.ss_proc = self.ss.serve_forever_process()
self.pp.start_pre_pool(2)
self.js.start_pool(2)
self.pp.start_post_pool(2)
self.congest_proc = self.congest.serve_forever_process()
def terminate(self):
logging.info('terminating all processes')
self.ss_proc.terminate()
self.congest_proc.terminate()
self.pp.terminate()
self.js.terminate()
def process_for(self, t):
"""Continue processing for t seconds.
"""
end = time.time() + t
while (time.time() < end):
self.process_all()
def process_all(self):
"""Continue processing packets if they are available.
If there is nothing to process, wait 0.01 seconds for
something to process. If nothing shows up then return.
"""
logging.debug('starting process all')
packets_processed = 0
should_wait = True
while (True):
did_work = False
if (self.ss.serve_once()):
did_work = True
if (self.pp.run_pre_once()):
did_work = True
if (self.js.serve_once()):
packets_processed += 1
did_work = True
if (self.pp.run_post_once()):
did_work = True
self.congest.run_once()
if did_work:
should_wait = True
continue
if not should_wait:
break
time.sleep(0.01)
should_wait = False
logging.debug('packets processed: %d', packets_processed)
td_error = self.ids[aid].critic.learn(
exp[0], exp[3], exp[1])
self.ids[aid].actor.learn(
exp[4], exp[2], td_error)
f_num += 1
self.memory['experience_pool_%d' % gpu_id] = experience_pool_new
e_e.set()
s_e.clear()
if __name__ == '__main__':
with tf.device('/cpu:0'):
game = Game()
manager = Manager()
memory = manager.dict()
states = collections.defaultdict(list)
for i in range(game.N):
for j in range(len(game.generator.matrix[i])):
if game.generator.matrix[i][j] == 1:
states[i].append(0)
node = game.ids[i]
for d in node.table:
states[i].append(0)
for d in node.table_peer:
states[i].append(0)
for d in node.table_provider:
states[i].append(0)
memory['states'] = states
def ecg_segmentation(data, parameters):
"""
Performs ECG segmentation on data according to specified parameters.
Input:
data (dict): input data.
example: data = {1: record 1 data, 2: record 2 data}
parameters (dict): filter parameters.
Example: parameters = {'model': 'engzee'}
Output:
output (dict): output data where keys correspond to record id numbers.
example: output = { 1: {'segments': record 1 signal segments, 'R': record 1 r peak indexes},
2: {'segments': record 2 signal segments, 'R': record 2 r peak indexes}}
Configurable fields:{"name": "??.??", "config": {}, "inputs": ["data", "parameters"], "outputs": ["output"]}
See Also:
Notes:
Example:
"""
if parameters['model'] == 'engzee':
model = ecgmodule.batch_engzee
else:
raise TypeError, "Model %s not implemented." % parameters['model']
# create work queue
work_queue = Queue()
manager = Manager()
output = manager.dict()
output['info'] = parameters
parameters.pop('model')
# fill queue
for recid in data.keys():
if recid == 'info': continue
work_queue.put({
'function': model,
'data': data.get(recid),
'parameters': parameters,
'recid': recid
})
# create N processes and associate them with the work_queue and do_work function
processes = [
Process(target=segmentation_work, args=(
work_queue,
output,
)) for _ in range(NUMBER_PROCESSES)
]
# lauch processes
for p in processes:
p.start()
# wait for processes to finish
print "waiting ..."
for p in processes:
p.join()
print "waiting is over"
for p in processes:
p.terminate()
return output
def _handle(self, context):
millis = context.args['duration']
time_range = DateTimeRangeModel(millis * 1000) # microseconds
return {'range': time_range}
"""
Resource endpoints.
We create a multiprocessing memory manager and shared
dict to enable multithreaded support.
This 'resource' data is used to test patching.
"""
BASE_RESOURCE = {'key': 'value'}
MANAGER = Manager()
RESOURCE = MANAGER.dict(BASE_RESOURCE)
class GetResourceEndpoint(Endpoint):
"""Returns the 'resource' as it exists in memory.
"""
_uri = "/resource"
_http_method = 'GET'
_route_name = "resource_get"
_returns = DebugResource("app resource.")
def _handle(self, context):
return dict(RESOURCE)
#consensus_output.write(str(element) + "\n")
with open(str(consensus_dict[element]),
"r") as consensus:
#next(consensus)
for line in consensus:
consensus_output.write(line)
#headings_output.close()
print sampleQueue
if __name__ == "__main__":
print "THIS IS MAIN FUNCTION"
manager = Manager()
freeze_support()
passed_dictionary = manager.dict()
consensus_string = filename + "_Quasi_strains.fasta"
finalconsensus_string = filename + "_final" + "_Quasi_strains.fasta"
clearing_string = open((filename + "_Quasi_strains.fasta"), "w")
clearing_string.close()
clearing_consensusstring = open(finalconsensus_string, "w")
clearing_consensusstring.close()
headings_string = filename + "_leaves"
finalheadings_string = filename + "_final" + "_leaves"
clearing_headingsstring = open(headings_string, "w")
clearing_headingsstring.close()
def run(self):
# -------------------------------------------------------------------------------------
# Set output data folder
self.sOutPath = setOutFolder(self.sFileName, self.sOutPath)
# Set land variable(s) information
self.oVarsDict = setVarsInfo(self.oVarsDict, self.sOutPath,
self.sDomainName)
# Info progress algorithm
self.progress()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Read filename and get DEM, LONGITUDE and LATITUDE data
QgsMessageLog.logMessage(' ==> GET DEM ... ', level=QgsMessageLog.INFO)
[
a2dVarData_DEM, a1oVarHeader, a2dVarData_GEOX, a2dVarData_GEOY,
dVarGeoXStep, dVarGeoYStep, dVarNoData
] = readRasterFile(self.sFileName)
a2dVarData_DEM_F = adjustVarPy2F(deepcopy(a2dVarData_DEM), 'float32')
QgsMessageLog.logMessage(' ==> GET DEM ... DONE!',
level=QgsMessageLog.INFO)
self.progress()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Compute MASK data
QgsMessageLog.logMessage(' ==> COMPUTE MASK ... ',
level=QgsMessageLog.INFO)
a2iVarData_MASK = computeMaskDomain(a2dVarData_DEM, dVarNoData)
QgsMessageLog.logMessage(' ==> COMPUTE MASK ... DONE!',
level=QgsMessageLog.INFO)
# Info progress algorithm
self.progress()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Compute CELL_AREA data
QgsMessageLog.logMessage(' ==> COMPUTE CELLAREA ... ',
level=QgsMessageLog.INFO)
a2dVarData_CAREA = computeCellArea(a2dVarData_GEOY, dVarGeoXStep,
dVarGeoYStep)
dVarAvg_CAREA = sqrt(nanmean(a2dVarData_CAREA))
a2dVarData_CAREA_F = adjustVarPy2F(deepcopy(a2dVarData_CAREA),
'float32')
QgsMessageLog.logMessage(' ==> COMPUTE CELLAREA ... DONE!',
level=QgsMessageLog.INFO)
# Info progress algorithm
self.progress()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Compute FLOW_DIRECTIONS data
QgsMessageLog.logMessage(' ==> COMPUTE FLOW DIRECTIONS ... ',
level=QgsMessageLog.INFO)
oProcManager = Manager()
oProcDict = oProcManager.dict()
self.oProcExc = Process(target=computeFlowDirections,
args=(a2dVarData_DEM_F, oProcDict))
self.oProcExc.start()
self.oProcExc.join()
a2iVarData_PNT_F = oProcDict['PNT']
a2iVarData_PNT = adjustVarF2Py(a2iVarData_PNT_F, 'int32')
QgsMessageLog.logMessage(' ==> COMPUTE FLOW DIRECTIONS ... DONE!',
level=QgsMessageLog.INFO)
# Info progress algorithm
self.progress()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Compute DRAINAGE AREA data
QgsMessageLog.logMessage(' ==> COMPUTE DRAINAGE AREA ... ',
level=QgsMessageLog.INFO)
oProcManager = Manager()
oProcDict = oProcManager.dict()
self.oProcExc = Process(target=computeDrainageArea,
args=(a2dVarData_DEM_F, a2iVarData_PNT_F,
oProcDict))
self.oProcExc.start()
self.oProcExc.join()
a2iVarData_DAREA_F = oProcDict['DAREA']
a2iVarData_DAREA = adjustVarF2Py(a2iVarData_DAREA_F, 'int32')
QgsMessageLog.logMessage(' ==> COMPUTE DRAINAGE AREA ... DONE!',
level=QgsMessageLog.INFO)
self.progress()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Compute CHANNELS NETWORK and PARTIAL DISTANCE
QgsMessageLog.logMessage(
' ==> COMPUTE CHANNELS NETWORK AND PARTIAL DISTANCE ... ',
level=QgsMessageLog.INFO)
oProcManager = Manager()
oProcDict = oProcManager.dict()
self.oProcExc = Process(target=computeChannelsNetwork,
args=(a2dVarData_DEM_F, a2iVarData_PNT_F,
a2iVarData_DAREA_F, dVarAvg_CAREA,
self.dThrAsk, oProcDict))
self.oProcExc.start()
self.oProcExc.join()
a2iVarData_CNET_F = oProcDict['CNET']
a2dVarData_PDIST_F = oProcDict['PDIST']
a2iVarData_CNET = adjustVarF2Py(a2iVarData_CNET_F, 'int32')
a2dVarData_PDIST = adjustVarF2Py(a2dVarData_PDIST_F, 'float32')
QgsMessageLog.logMessage(
' ==> COMPUTE CHANNELS NETWORK AND PARTIAL DISTANCE ... DONE!',
level=QgsMessageLog.INFO)
self.progress()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Compute ALPHA and BETA data
QgsMessageLog.logMessage(' ==> COMPUTE ALPHA AND BETA ... ',
level=QgsMessageLog.INFO)
oProcManager = Manager()
oProcDict = oProcManager.dict()
self.oProcExc = Process(target=computeWatertableSlopes,
args=(a2dVarData_DEM_F, a2iVarData_PNT_F,
a2iVarData_CNET_F, a2iVarData_DAREA_F,
self.dThrWts, oProcDict))
self.oProcExc.start()
self.oProcExc.join()
a2dVarData_ALPHA_F = oProcDict['ALPHA']
a2dVarData_BETA_F = oProcDict['BETA']
a2dVarData_ALPHA = adjustVarF2Py(a2dVarData_ALPHA_F, 'float64')
a2dVarData_BETA = adjustVarF2Py(a2dVarData_BETA_F, 'float64')
QgsMessageLog.logMessage(' ==> COMPUTE ALPHA AND BETA ... DONE!',
level=QgsMessageLog.INFO)
self.progress()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Compute COEFF RESOLUTION data
QgsMessageLog.logMessage(' ==> COMPUTE COEFF RESOLUTION ... ',
level=QgsMessageLog.INFO)
oProcManager = Manager()
oProcDict = oProcManager.dict()
self.oProcExc = Process(target=computeCoeffResolution,
args=(a2dVarData_DEM_F, a2iVarData_DAREA_F,
a2iVarData_CNET_F, a2dVarData_CAREA_F,
self.dThrCRes, oProcDict))
self.oProcExc.start()
self.oProcExc.join()
a2dVarData_CRES_F = oProcDict['CRES']
a2dVarData_CRES = adjustVarF2Py(a2dVarData_CRES_F, 'float32')
QgsMessageLog.logMessage(' ==> COMPUTE COEFF RESOLUTION ... DONE!',
level=QgsMessageLog.INFO)
self.progress()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Update variable(s) in a common workspace
oDictVarName = {
'Var_01': sVarName_DEM,
'Var_02': sVarName_GEOX,
'Var_03': sVarName_GEOY,
'Var_04': sVarName_MASK,
'Var_05': sVarName_CAREA,
'Var_06': sVarName_PNT,
'Var_07': sVarName_DAREA,
'Var_08': sVarName_CNET,
'Var_09': sVarName_PDIST,
'Var_10': sVarName_ALPHA,
'Var_11': sVarName_BETA,
'Var_12': sVarName_CRES,
}
oDictVarValue = {
'Var_01': a2dVarData_DEM,
'Var_02': a2dVarData_GEOX,
'Var_03': a2dVarData_GEOY,
'Var_04': a2iVarData_MASK,
'Var_05': a2dVarData_CAREA,
'Var_06': a2iVarData_PNT,
'Var_07': a2iVarData_DAREA,
'Var_08': a2iVarData_CNET,
'Var_09': a2dVarData_PDIST,
'Var_10': a2dVarData_ALPHA,
'Var_11': a2dVarData_BETA,
'Var_12': a2dVarData_CRES
}
# Update variable(s) information
self.oVarsDict = updateVarsInfo(self.oVarsDict, oDictVarName,
oDictVarValue, 'DataValue')
self.oVarsHeader = a1oVarHeader
from warnings import simplefilter
from functools import partial
from cryptoalgotrading.riskmanagement import RiskManagement
from cryptoalgotrading.riskmanagement import Binance as Bnb
from multiprocessing import Pool, Manager
from cryptoalgotrading.aux import get_markets_list, \
log, Bittrex, stop_loss, trailing_stop_loss, \
timeit, safe, connect_db, get_markets_on_files, \
manage_files, num_processors, plot_data, \
check_market_name, get_data_from_file, \
time_to_index, get_historical_data, \
binance2btrx
import logging as log
manager = Manager()
cached = manager.dict()
def signal_handler(sig, frame):
log.info(f"You pressed Ctrl+C!")
sys.exit(0)
# Prevents FutureWarning from Pandas.
simplefilter(action='ignore', category=FutureWarning)
@safe
def is_time_to_exit(data,
funcs,
smas=var.default_smas,
def execute(self):
#import the algorithm module
try:
importStr = 'from algorithm.rating.' + self.config[
'recommender'] + ' import ' + self.config['recommender']
exec(importStr)
except ImportError:
importStr = 'from algorithm.ranking.' + self.config[
'recommender'] + ' import ' + self.config['recommender']
exec(importStr)
if self.evaluation.contains('-cv'):
k = int(self.evaluation['-cv'])
if k <= 1 or k > 10:
k = 3
mkl.set_num_threads(max(1, mkl.get_max_threads() / k))
#create the manager used to communication in multiprocess
manager = Manager()
m = manager.dict()
i = 1
tasks = []
binarized = False
if self.evaluation.contains('-b'):
binarized = True
for train, test in DataSplit.crossValidation(self.trainingData,
k,
binarized=binarized):
fold = '[' + str(i) + ']'
if self.config.contains('social'):
recommender = self.config[
'recommender'] + "(self.config,train,test,self.relation,fold)"
else:
recommender = self.config[
'recommender'] + "(self.config,train,test,fold)"
#create the process
p = Process(target=run, args=(m, eval(recommender), i))
tasks.append(p)
i += 1
#start the processes
for p in tasks:
p.start()
#wait until all processes are completed
for p in tasks:
p.join()
#compute the mean error of k-fold cross validation
self.measure = [dict(m)[i] for i in range(1, k + 1)]
res = []
for i in range(len(self.measure[0])):
measure = self.measure[0][i].split(':')[0]
total = 0
for j in range(k):
total += float(self.measure[j][i].split(':')[1])
res.append(measure + ':' + str(total / k) + '\n')
#output result
currentTime = strftime("%Y-%m-%d %H-%M-%S", localtime(time()))
outDir = LineConfig(self.config['output.setup'])['-dir']
fileName = self.config[
'recommender'] + '@' + currentTime + '-' + str(
k) + '-fold-cv' + '.txt'
FileIO.writeFile(outDir, fileName, res)
print 'The result of %d-fold cross validation:\n%s' % (
k, ''.join(res))
else:
if self.config.contains('social'):
recommender = self.config[
'recommender'] + '(self.config,self.trainingData,self.testData,self.relation)'
else:
recommender = self.config[
'recommender'] + '(self.config,self.trainingData,self.testData)'
eval(recommender).execute()
def __init__(self):
manager = Manager()
self.non_finding = manager.dict()
self.crash = manager.dict()
self.timeout = manager.dict()
self.kasan = manager.dict()
def kicost(in_file,
eda_tool_name,
out_filename,
user_fields,
ignore_fields,
group_fields,
variant,
dist_list=list(distributor_dict.keys()),
num_processes=4,
scrape_retries=5,
throttling_delay=0.0,
collapse_refs=True,
local_currency='USD'):
''' @brief Run KiCost.
Take a schematic input file and create an output file with a cost spreadsheet in xlsx format.
@param in_file `list(str())` List of the names of the input BOM files.
@param eda_tool_name `list(str())` of the EDA modules to be used to open the `in_file`list.
@param out_filename `str()` XLSX output file name.
@param user_fields `list()` of the user fields to be included on the spreadsheet global part.
@param ignore_fields `list()` of the fields to be ignored on the read EDA modules.
@param group_fields `list()` of the fields to be grouped/merged on the function group parts that
are not grouped by default.
@param variant `list(str())` of regular expression to the BOM variant of each file in `in_file`.
@param dist_list `list(str())` to be scraped, if empty will be scraped with all distributors
modules. If `None`, no web/local distributors will be scraped.
@param num_processes `int()` Number of parallel processes used for web scraping part data. Use
1 for serial mode.
@param scrape_retries `int()` Number of attempts to retrieve part data from a website..
@param throttling_delay `float()` Minimum delay (in seconds) between successive accesses to a
distributor's website.
@param collapse_refs `bool()` Collapse or not the designator references in the spreadsheet.
Default `True`.
@param local_currency `str()` Local/country in ISO3166:2 and currency in ISO4217. Default 'USD'.
'''
# Only keep distributors in the included list and not in the excluded list.
if dist_list != None:
if not dist_list:
dist_list = list(distributor_dict.keys())
if not 'local_template' in dist_list:
dist_list += ['local_template'
] # Needed later for creating non-web distributors.
for d in list(distributor_dict.keys()):
if not d in dist_list:
distributor_dict.pop(d, None)
else:
for d in list(distributor_dict.keys()):
distributor_dict.pop(d, None)
# Deal with some code exception (only one EDA tool or variant
# informed in the multiple BOM files input).
if not isinstance(in_file, list):
in_file = [in_file]
if not isinstance(variant, list):
variant = [variant] * len(in_file)
elif len(variant) != len(in_file):
variant = [variant[0]] * len(in_file) #Assume the first as default.
if not isinstance(eda_tool_name, list):
eda_tool_name = [eda_tool_name] * len(in_file)
elif len(eda_tool_name) != len(in_file):
eda_tool_name = [eda_tool_name[0]] * len(
in_file) #Assume the first as default.
# Get groups of identical parts.
parts = dict()
prj_info = list()
for i_prj in range(len(in_file)):
eda_tool_module = eda_modules[eda_tool_name[i_prj]]
p, info = eda_tool_module.get_part_groups(in_file[i_prj],
ignore_fields,
variant[i_prj])
p = subpartqty_split(p)
# In the case of multiple BOM files, add the project prefix identifier
# to each reference/designator. Use the field 'manf#_qty' to control
# each quantity goes to each project creating a `list()` with length
# of number of BOM files. This vector will be used in the `group_parts()`
# to create groups with elements of same 'manf#' that came for different
# projects.
if len(in_file) > 1:
logger.log(
DEBUG_OVERVIEW,
'Multi BOMs detected, attaching project identification to references...'
)
qty_base = ['0'] * len(in_file) # Base zero quantity vector.
for p_ref in list(p.keys()):
try:
qty_base[i_prj] = p[p_ref]['manf#_qty']
except:
qty_base[i_prj] = '1'
p[p_ref]['manf#_qty'] = qty_base.copy()
p['prj' + str(i_prj) + SEPRTR + p_ref] = p.pop(p_ref)
parts.update(p.copy())
prj_info.append(info.copy())
# Group part out of the module to be possible to merge different
# project lists, ignore some field to merge given in the `group_fields`.
FIELDS_SPREADSHEET = [
'refs', 'value', 'desc', 'footprint', 'manf', 'manf#'
]
FIELDS_MANFCAT = ([d + '#' for d in distributor_dict] + ['manf#'])
FIELDS_MANFQTY = ([d + '#_qty' for d in distributor_dict] + ['manf#_qty'])
FIELDS_IGNORE = FIELDS_SPREADSHEET + FIELDS_MANFCAT + FIELDS_MANFQTY + user_fields + [
'pricing'
]
for ref, fields in list(parts.items()):
for f in fields:
# Merge all extra fields that read on the files that will
# not be displayed (Needed to check `user_fields`).
if f not in FIELDS_IGNORE and SEPRTR not in f and not f in group_fields: # Not include repetitive filed names or fields with the separator `:` defined on `SEPRTR`.
group_fields += [f]
# Some fields to be merged on specific EDA are enrolled bellow.
if 'kicad' in eda_tool_name:
group_fields += [
'libpart'
] # This field may be a mess on multiple sheet designs.
if len(set(eda_tool_name)) > 2:
# If more than one EDA software was used, ignore the 'footprint'
# field, because they could have different libraries names.
group_fields += ['footprint']
group_fields += ['desc', 'var'] # Always ignore 'desc' ('description')
# and 'var' ('variant') fields, merging
# the components in groups.
group_fields = set(group_fields)
parts = group_parts(parts, group_fields)
# If do not have the manufacture code 'manf#' and just distributors codes,
# check if is asked to scrap a distributor that do not have any code in the
# parts so, exclude this distributors for the scrap list. This decrease the
# warning messages given during the process.
all_fields = []
for p in parts:
all_fields += list(p.fields.keys())
all_fields = set(all_fields)
if not 'manf#' in all_fields:
dist_not_rmv = [
d for d in distributor_dict.keys() if d + '#' in all_fields
]
dist_not_rmv += ['local_template'
] # Needed later for creating non-web distributors.
#distributor_scrap = {d:distributor_dict[d] for d in dist_not_rmv}
distributors = distributor_dict.copy().keys()
for d in distributors:
if not d in dist_not_rmv:
logger.warning(
"No 'manf#' and '%s#' field in any part: distributor '%s' will be not scraped.",
d, distributor_dict[d]['label'])
distributor_dict.pop(d, None)
# Create an HTML page containing all the local part information.
local_part_html = create_local_part_html(parts, distributor_dict)
if logger.isEnabledFor(DEBUG_DETAILED):
pprint.pprint(distributor_dict)
# Get the distributor product page for each part and scrape the part data.
if dist_list:
if local_currency:
logger.log(
DEBUG_OVERVIEW,
'# Configuring the distributors locale and currency...')
if num_processes <= 1:
for d in distributor_dict:
config_distributor(distributor_dict[d]['module'],
local_currency)
else:
logger.log(
DEBUG_OBSESSIVE, 'Using {} simultaneous access...'.format(
min(len(distributor_dict), num_processes)))
pool = Pool(num_processes)
for d in distributor_dict:
args = [distributor_dict[d]['module'], local_currency]
pool.apply_async(config_distributor, args)
pool.close()
pool.join()
logger.log(DEBUG_OVERVIEW,
'# Scraping part data for each component group...')
# Set the throttling delay for each distributor.
for d in distributor_dict:
distributor_dict[d]['throttling_delay'] = throttling_delay
global scraping_progress
scraping_progress = tqdm.tqdm(desc='Progress',
total=len(parts),
unit='part',
miniters=1)
# Change the logging print channel to `tqdm` to keep the process bar to the end of terminal.
class TqdmLoggingHandler(logging.Handler):
'''Overload the class to write the logging through the `tqdm`.'''
def __init__(self, level=logging.NOTSET):
super(self.__class__, self).__init__(level)
def emit(self, record):
try:
msg = self.format(record)
tqdm.tqdm.write(msg)
self.flush()
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
logger.addHandler(TqdmLoggingHandler())
if num_processes <= 1:
# Scrape data, one part at a time using single processing.
class DummyLock:
"""Dummy synchronization lock used when single processing."""
def __init__(self):
pass
def acquire(*args, **kwargs):
return True # Lock can ALWAYS be acquired when just one process is running.
def release(*args, **kwargs):
pass
# Create sync lock and timeouts to control the rate at which distributor
# websites are scraped.
throttle_lock = DummyLock()
throttle_timeouts = dict()
throttle_timeouts = {d: time() for d in distributor_dict}
for i in range(len(parts)):
args = (i, parts[i], distributor_dict, local_part_html,
scrape_retries, logger.getEffectiveLevel(),
throttle_lock, throttle_timeouts)
id, url, part_num, price_tiers, qty_avail, info_dist = scrape_part(
args)
parts[id].part_num = part_num
parts[id].url = url
parts[id].price_tiers = price_tiers
parts[id].qty_avail = qty_avail
parts[id].info_dist = info_dist # Extra distributor web page.
scraping_progress.update(1)
else:
# Scrape data, multiple parts at a time using multiprocessing.
# Create sync lock and timeouts to control the rate at which distributor
# websites are scraped.
throttle_manager = Manager() # Manages shared lock and `dict`.
throttle_lock = throttle_manager.Lock()
throttle_timeouts = throttle_manager.dict()
for d in distributor_dict:
throttle_timeouts[d] = time()
# Create pool of processes to scrape data for multiple parts simultaneously.
pool = Pool(num_processes)
# Package part data for passing to each process.
arg_sets = [(i, parts[i], distributor_dict, local_part_html,
scrape_retries, logger.getEffectiveLevel(),
throttle_lock, throttle_timeouts)
for i in range(len(parts))]
# Define a callback routine for updating the scraping progress bar.
def update(x):
scraping_progress.update(1)
return x
# Start the web scraping processes, one for each part.
logger.log(
DEBUG_OBSESSIVE,
'Starting {} parallels process to scrap parts...'.format(
num_processes))
results = [
pool.apply_async(scrape_part, [args], callback=update)
for args in arg_sets
]
# Wait for all the processes to have results, then kill-off all the scraping processes.
for r in results:
while (not r.ready()):
pass
logger.log(DEBUG_OVERVIEW,
'All parallels process finished with success.')
pool.close()
pool.join()
# Get the data from each process result structure.
for result in results:
id, url, part_num, price_tiers, qty_avail, info_dist = result.get(
)
parts[id].part_num = part_num
parts[id].url = url
parts[id].price_tiers = price_tiers
parts[id].qty_avail = qty_avail
parts[id].info_dist = info_dist # Extra distributor web page.
# Done with the scraping progress bar so delete it or else we get an
# error when the program terminates.
logger.removeHandler(
TqdmLoggingHandler()) # Return the print channel of the logging.
del scraping_progress
# Create the part pricing spreadsheet.
create_spreadsheet(parts, prj_info, out_filename, collapse_refs,
user_fields,
'-'.join(variant) if len(variant) > 1 else variant[0])
# Print component groups for debugging purposes.
if logger.isEnabledFor(DEBUG_DETAILED):
for part in parts:
for f in dir(part):
if f.startswith('__'):
continue
elif f.startswith('html_trees'):
continue
else:
print('{} = '.format(f), end=' ')
try:
pprint.pprint(part.__dict__[f])
except TypeError:
# Python 2.7 pprint has some problem ordering None and strings.
print(part.__dict__[f])
except KeyError:
pass
print()
