VERSION_NONE = -1 # Data directories ROOT_DIR = '/mnt' DEST_DIR = ROOT_DIR + '/idseq/data' # Generated data go here REF_DIR = ROOT_DIR + '/idseq/ref' # Reference genome / ref databases go here OUTPUT_VERSIONS = [] print_lock = multiprocessing.RLock() # Peak network and storage perf for a typical small instance is saturated by # just a few concurrent streams. MAX_CONCURRENT_COPY_OPERATIONS = 8 iostream = multiprocessing.Semaphore(MAX_CONCURRENT_COPY_OPERATIONS) # Make a second semaphore for uploads to reserve some capacity for downloads. MAX_CONCURRENT_UPLOAD_OPERATIONS = 4 iostream_uploads = multiprocessing.Semaphore(MAX_CONCURRENT_UPLOAD_OPERATIONS) # Definitions for integration with web app TAX_LEVEL_SPECIES = 1 TAX_LEVEL_GENUS = 2 TAX_LEVEL_FAMILY = 3 NULL_SPECIES_ID = -100 NULL_GENUS_ID = -200 NULL_FAMILY_ID = -300 NULL_LINEAGE = (str(NULL_SPECIES_ID), str(NULL_GENUS_ID), str(NULL_FAMILY_ID)) JOB_SUCCEEDED = "succeeded" JOB_FAILED = "failed"
def makeInactive(self, name):
with self.lock:
self.active.remove(name)
def __str__(self):
with self.lock:
return str(self.active)
def worker(s, pool):
name = multiprocessing.current_process().name
with s:
pool.makeActive(name)
print 'Now running: %s' % str(pool)
time.sleep(random.randint(5, 10))
pool.makeInactive(name)
if __name__ == '__main__':
pool = ActivePool()
s = multiprocessing.Semaphore(3)
jobs = [
multiprocessing.Process(target=worker, args=(s, pool), name=str(i))
for i in range(10)
]
for j in jobs:
j.start()
for j in jobs:
j.join()
print 'Now running: %s' % str(pool)
import requests
import math
from datetime import datetime
import pathlib
import timeit
import time
import sys
import multiprocessing
import utils
if __name__ == '__main__':
procs = []
num_process = min(multiprocessing.cpu_count() - 1, 8)
sema = multiprocessing.Semaphore(num_process)
exclude_list_endswith = [
'스팩', '리츠', '증권', '은행', '홀딩스', '지주', '건설', '화재', '종금', '캐피탈', '투자'
]
exclude_list_exact = [
'한국테크놀로지그룹', '인터파크', '아세아', 'CJ', 'LG', '경동인베스트', '엘브이엠씨', '대웅',
'아모레퍼시픽그룹', '지투알', 'BGF', '코오롱', 'GS', 'SK', '한화', '현대모비스', 'DL',
'HDC', '효성', '동원개발', '금호산업', '휴온스글로벌'
]
#holdings
exclude_list_contain = ['스팩']
required_ror_percent = utils.get_required_rate_of_return()
path = str(pathlib.Path().absolute()) + '\\'
set_parts[site][content].add(taxon)
# Remove ignored characters from set_parts
for c in ignored_chars:
removed = set_parts[site].pop(c, None)
if set_parts[site] == {}:
print("Error: Empty character alignment at position " + str(site) +
". TIGER rates not calculated.",
file=sys.stderr)
exit(1)
# Steps 2 and 3: calculate partition agreements and TIGER rates concurrently
pool = ActivePool()
pool.data.update(char_dict)
s = multiprocessing.Semaphore(args.n_processes)
jobs = [
multiprocessing.Process(target=calculate_tiger_rates,
name=str(k),
args=(k, pool))
for k in split_list(list(char_dict.keys()), args.n_processes)
]
for j in jobs:
j.start()
for j in jobs:
j.join()
for k in sorted(pool.result.keys()):
line = ""
def CreateModelDiagrams(M, options):
# This function is a "master", calling many other functions based on command
# line input. Other than code cleanliness, there is no reason that the
# logic couldn't be in main()
# if the user has listed more than one dot_dat, arbitrarily choose the first
# as the name of this run.
datname = os.path.basename(options.dot_dat[0])[:-4]
images_dir = "images_" + datname
if os.path.exists(images_dir):
rmtree(images_dir)
os.mkdir(images_dir)
os.chdir(images_dir)
os.makedirs('commodities')
os.makedirs('processes')
os.makedirs('results')
##############################################
#MAIN MODEL AND RESULTS AND EVERYTHING ELSE
kwargs = dict(
model=M,
images_dir='images_%s' % datname,
image_format=options.graph_format.lower(),
options=options,
tech_color='darkseagreen',
commodity_color='lightsteelblue',
unused_color='powderblue',
arrowheadout_color='forestgreen',
arrowheadin_color='firebrick',
usedfont_color='black',
unusedfont_color='chocolate',
menu_color='hotpink',
home_color='gray75',
#MODELDETAILED,
md_tech_color='hotpink',
#SUBGRAPHS (option 1),
sb_incom_color='lightsteelblue',
sb_outcom_color='lawngreen',
sb_vpbackg_color='lightgrey',
sb_vp_color='white',
sb_arrow_color='forestgreen',
#SUBGRAPH 1 ARROW COLORS
# feel free to add more colors here
color_list=('red', 'orange', 'gold', 'green', 'blue', 'purple',
'hotpink', 'cyan', 'burlywood', 'coral', 'limegreen',
'black', 'brown'),
)
####################################
# Do all necessary work in parallel, taking advantage of whatever cores
# are available to the computer on which this code is run. To add a
# function to the pool, ensure that the function is in the 'gvizFunctions'
# tuple below, and that the function has all it needs to work passed in
# via the 'kwargs' dict above.
gvizFunctions = (
CreateCompleteEnergySystemDiagram,
CreateCommodityPartialGraphs,
CreateProcessPartialGraphs,
CreateMainModelDiagram,
CreateTechResultsDiagrams,
CreateCommodityPartialResults,
CreateMainResultsDiagram,
CreatePartialSegmentsDiagram,
)
if 'win' in sys.platform:
msg = ('\n\nRunning in Windows ... Temoa is currently unable to use '
'multiple processes. Generating graphs will take a bit longer.'
'\n')
SE.write(msg)
for func in gvizFunctions:
func(**kwargs)
else:
sem = MP.Semaphore(MP.cpu_count())
def do_work(func):
sem.acquire()
func(**kwargs)
sem.release()
processes = [
MP.Process(target=do_work, args=(func, )) for func in gvizFunctions
]
for p in processes:
p.start()
for p in processes:
p.join()
os.chdir('..')
def _build_parallel_ctrl(self, n_worker):
super()._build_parallel_ctrl(n_worker)
self.sync.obs_ready = [mp.Semaphore(0) for _ in range(n_worker)]
self.sync.act_ready = [mp.Semaphore(0) for _ in range(n_worker)]
def main():
global flagTerminate
global beanstalk
global logger
logger = setupLogger()
# define a Handler which writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger().addHandler(console)
random.seed()
pool = ActivePool()
assignments = pool.mgr.dict()
count = pool.mgr.Value('i', 0)
ADBFAILCOUNT = pool.mgr.Value('i', 0)
VMs = pool.mgr.dict()
s = multiprocessing.Semaphore(NUMACTIVEWORKERS)
q = multiprocessing.Queue()
r = multiprocessing.Queue()
results = []
al = multiprocessing.RLock()
vml = multiprocessing.RLock()
cl = multiprocessing.RLock()
get_VMs(VMs)
list_VMs(VMs)
#make sure any pcaps are killed and we are starting with a clean filesystem
for i in range(30):
#q.put("item " + str(i) + " content")
q.put("live" + str(i))
#adb ports are 5554 - 5584, only even
port = ADBPORT
jobs = [
multiprocessing.Process(target=worker,
name=str(i),
args=(s, pool, q, "hi", VMs, vml,
str(port + (i * 2)), count, al, r,
ADBFAILCOUNT)) for i in range(NUMWORKERS)
]
for j in jobs:
logger.info("starting worker %s" % j.name)
j.start()
# for j in jobs:
# j.join()
# logger.info( 'Now running: %s' % str(pool))
#make the connection after the child spawning, each child makes thier own connections
rebeanstalk()
while True:
logger.info("==============================")
logger.info("processed %d samples so far" % count.value)
try:
#not great, but the parent is the only consumer of r items
thing = r.get(block=False)
results.append(thing)
except:
logger.warning("results queue exception")
if len(results) > 0:
logger.info("average sample time: %s" %
str(sum(results) / float(len(results))))
try:
if BSMSBOXQ not in beanstalk.tubes():
logger.info("there are no samples in the beanstalk")
else:
if beanstalk.peek_ready() is not None:
logger.info(
str(
beanstalk.stats_tube(BSMSBOXQ)
['current-jobs-ready']) +
" samples ready to process")
else:
logger.info("0 samples ready to process")
if beanstalk.peek_reserved() is not None:
logger.info(
str(
beanstalk.stats_tube(BSMSBOXQ)
['current-jobs-reserved']) +
" samples are processing")
else:
logger.info("0 samples are processing")
if beanstalk.peek_delayed() is not None:
logger.info(
str(
beanstalk.stats_tube(BSMSBOXQ)
['current-jobs-delayed']) + " samples are delayed")
else:
logger.info("0 samples are delayed")
except KeyError:
logger.warning("beanstalk keyerror")
except:
logger.warning("beanstalk general error")
logger.info("about " + str(q.qsize()) + " current q items")
logger.info("adb has failed " + str(ADBFAILCOUNT.value) +
" times since last kick")
logger.debug("should be " + str(NUMWORKERS) + " processes:")
walive = 0
wdead = 0
for j in jobs:
if j.is_alive():
logger.debug(" " + str(j.pid))
walive += 1
else:
logger.debug(" dead")
wdead += 1
#this will kill the controller, leaving the children pipes wanting more (you will get sigpipe errors)
#this is pretty harsh and shouldn't probably be done in production
#TODO
#sys.exit(1)
logger.info("%s alive; %s dead" % (walive, wdead))
updateStatus(NUMWORKERS, walive, wdead)
list_VMs(VMs)
time.sleep(VM_POLL_REST + 10)
#blocks until all items in q are processed
#safe to exit
#should check for sigint
#if q.qsize() == 0:
# for i in jobs:
# q.put("die")
# q.close()
sys.exit(0)
signal.signal(signal.SIGINT, sigint_handler)
syslog.syslog('Startup - Entering main polling loop...')
while True:
get_VMs(VMs)
list_VMs(VMs)
#make sure any pcaps are killed and we are starting with a clean filesystem
#pcap_terminate(m)
#job_cleanup(m)
#check for a job in the beanstalk Q
if not beanstalk:
error_log('Re-establishing connection to beanstalkd')
beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT)
beanstalk.watch(BSMSBOXQ)
beanstalk.use(BSMSBOXQ)
beanstalk.ignore('default')
#we want to reserve, not peak_ready because if we can't process, we want to release with a delay later
beanstalkjob = beanstalk.reserve()
#if there was a job and we got it then queue up analysis
if beanstalkjob:
try:
job = eval(beanstalkjob.body)
except:
error_log('ERROR: could not eval job - invalid job')
beanstalkjob.delete()
return False
#is there a VM compatible with this sample that is ready to be used?
m = findCompatibleVM(job, VMs, vml)
if m is not None:
logger.info("yay using VM " + str(m))
#process sample
sys.exit(0)
#delete from queue if successfull
#beanstalkjob.delete()
else:
#no VM found, put it back in the queue (nobody will process for 60 seconds
beanstalkjob.release(delay=60)
logger.info("requeuing job requiring a target version " +
job['target'])
#
# #break ot of the loop and exit if the assignments are clear and the terminate flag is set
if flagTerminate and len(assignments) == 0:
syslog.syslog('All assigned jobs are completed (EXITING)...')
break
#put a pause in the polling cycle...set to 0 for no pause
time.sleep(VM_POLL_REST)
sys.exit(0)
syslog.syslog('Shutdown - Leaving main polling loop...')
return 0
sem1.release()
def cons2(Q2, sem1, sem2):
n = 0
while n != -1:
sem1.acquire()
n = Q2.get()
print(n)
sem2.release()
Q1 = mp.Queue()
Q2 = mp.Queue()
sem1 = mp.Semaphore(1)
sem2 = mp.Semaphore(0)
P1 = mp.Process(target=prod1, args=(Q1, ))
P2 = mp.Process(target=prod2, args=(Q2, ))
C1 = mp.Process(target=cons1, args=(Q1, sem1, sem2))
C2 = mp.Process(target=cons2, args=(Q2, sem1, sem2))
P1.start()
P2.start()
C1.start()
C2.start()
def grow(self, nsims=10, gamma=0.9, parallelize=False):
"""
Grow Tree by one node
gamma between 0.5 and 1
"""
### SELECTION
start_node = self.root_id
# Sample actions of other robots
# NOTE: Sampling done at the begining for dependency graph reasons
state = self._get_system_state(start_node)
# Propagate down the tree
# check how _select handles mu, N = 0
while len(self._childNodes(start_node)) > 0:
start_node = self._select(self._childNodes(start_node))
### EXPANSION
# check if _expansion changes start_node to the node after jumping
self._expansion(start_node)
# print("Grow!")
### SIMULATION
avg_reward = 0
best_reward = float("-Inf")
best_rollout = None
# Parallization
if parallelize:
processes = []
pipe_list = []
mp_context = multiprocessing.get_context("fork")
sema = multiprocessing.Semaphore(12)
for i in range(nsims):
recv_end, send_end = mp_context.Pipe(False)
p = mp_context.Process(target=self._simulate,
args=(start_node, state, send_end,
sema))
processes.append(p)
pipe_list.append(recv_end)
p.start()
for i in range(nsims):
if parallelize:
recvd = pipe_list[i].recv()
processes[i].join()
else:
recvd = self._simulate(start_node, state)
rew = recvd["reward"]
temp_state = recvd["temp_state"]
avg_reward += rew
# new_return = 0
# for j in reversed(range(rew.len())):
# new_return = rew[j] + gamma * new_return
# avg_reward += new_return
# if best reward so far, store the rollout in the new node
if rew > best_reward:
best_reward = rew
best_rollout = copy(temp_state)
avg_reward = avg_reward / nsims
self.graph.nodes[start_node]["mu"] = avg_reward
self.graph.nodes[start_node]["best_reward"] = best_reward
self.graph.nodes[start_node]["N"] = 1
self.graph.nodes[start_node]["best_rollout"] = deepcopy(best_rollout)
### BACKPROPOGATION
while start_node != 1: # while not root node
start_node = self._parent(start_node)
self.graph.nodes[start_node]["mu"] = \
(gamma * self.graph.nodes[start_node]["mu"] * \
self.graph.nodes[start_node]["N"] + avg_reward) \
/ (self.graph.nodes[start_node]["N"] + 1)
self.graph.nodes[start_node]["N"] = \
gamma * self.graph.nodes[start_node]["N"] + 1
if best_reward > self.graph.nodes[start_node]["best_reward"]:
self.graph.nodes[start_node]["best_reward"] = best_reward
self.graph.nodes[start_node]["best_rollout"] = deepcopy(
best_rollout)
self._update_distribution()
return avg_reward
import time
import select
from qalib.qabase.exceptions import ConnectionError
logging.getLogger('paramiko').setLevel(logging.WARNING)
log = logging.getLogger(__name__)
if not log.handlers:
log.addHandler(logging.NullHandler())
DEFAULT_SHELL_MATCH_TIMEOUT = 60
INTERACTIVE_SHELL_TIMEOUT = 60
OPEN_SESSION_TIMEOUT = 10
sema = multiprocessing.Semaphore(9)
def _open_session_with_timeout(transport):
"""
Helper function which encapsulates a difference between older and
newer Paramiko versions.
For a given paramiko.Transport object, returns a session opened
from the tranport, with a timeout of OPEN_SESSION_TIMEOUT seconds.
New versions have a timeout parameter when opening a session, so
it's easy. But on old versions, the timeout is very long and not
configurable, so it can take us a long time to fail over if we catch
the node just as it goes down. So we use a background thread to
emulate this timeout.
print("Je suis le process E")
s2.acquire()
s2.acquire()
print("Je suis le process E et je vais generer 1 jeton dans 5 sec")
time.sleep(5)
s3.release()
sys.exit(0)
def processF(s3):
print("Je suis le process F")
s3.acquire()
s3.acquire()
print("Je suis le process F et je suis DEBLOQUE")
sys.exit(0)
sem = mp.Semaphore(0)
sem2 = mp.Semaphore(0)
sem3 = mp.Semaphore(0)
pA = mp.Process(target = processA, args =(sem,))
pB = mp.Process(target = processB, args =(sem,sem2,))
pC = mp.Process(target = processC, args =(sem,sem2,))
pD = mp.Process(target = processD, args =(sem,sem3,))
pE = mp.Process(target = processE, args =(sem2,sem3,))
pF = mp.Process(target = processF, args =(sem3,))
pA.start()
pB.start()
pC.start()
pD.start()
pE.start()
pF.start()
pA.join()
multiprocessing.Process.__init__(self)
self.loop = loop
self.lock = lock
def run(self):
for count in range(self.loop):
time.sleep(0.1)
self.lock.acquire()
print 'Pid: ' + str(self.pid) + ' LoopCount: ' + str(count)
self.lock.release()
#############################
###Semaphore#######
buffer1 = multiprocessing.Queue(10)
empty = multiprocessing.Semaphore(2)
full = multiprocessing.Semaphore(0)
lock = multiprocessing.Lock()
class Consumer(multiprocessing.Process):
"""docstring for Consumer"""
def __init__(self, pipe):
multiprocessing.Process.__init__(self)
self.pipe = pipe
def run(self):
self.pipe.send('Consumer Words')
print 'Consumer Received:', self.pipe.recv()
'''global buffer1, empty, full, lock
while True:
def __init__(self):
self.process = mp.Process
self.mgr = mp.Manager()
self.sem = mp.Semaphore(mp.cpu_count())
self.result = self.mgr.dict() #result dict
self.ppipe, self.cpipe = mp.Pipe() #parent pipe, #child pipe
import random
import multiprocessing as mp
from multiprocessing import Process
# from multiprocessing import Queue # ne fonctionne pas en python 3
import queue
permission_de_prendre_un_cookie = mp.Semaphore(1)
def put_random_int(q: queue):
print("Start put_random_int")
nombre_production = 10
cpt = 0
while cpt < nombre_production:
q.put(random.randint(0, 20))
cpt = cpt + 1
print("End put_random_int")
def get_random_int(q: queue):
print("Start get_random_int")
permission_de_prendre_un_cookie.acquire()
cookie = q.get()
print(f"j'ai pris ce cookie: {cookie}")
permission_de_prendre_un_cookie.release()
print("End get_random_int")
if __name__ == "__main__":
print("Start main")
q1 = queue.Queue()
# -*- coding: utf-8 -*-
"""
Created on Fri Jun 8 08:37:18 2018
@author: antoine.bralet
"""
import multiprocessing as mp
variable_partagee = mp.Value("i", 0) # ce sera un entier initialisé à0
verrou = mp.Semaphore() # Val init=1
def count2_SC_sem(nb_iterations):
""" Chacun écrit àson rythme (non protégée)"""
global variable_partagee
for i in range(nb_iterations):
verrou.acquire()
variable_partagee.value += 1
verrou.release()
def test_SC_protege_par_Sem():
# if __name__ == ’__main__’ :
nb_iterations = 5000
# La variable partagée : placée hors cette fonction (sinon, la passr en param)
# variable_partagee = mp.Value(’i’,0) # ce sera un entier initialisé à0
print("la valeur de variable_partagee AVANT les incrémentations : ",
variable_partagee.value)
# On crée 2 process
pid1 = mp.Process(target=count2_SC_sem, args=(nb_iterations, ))
pid1.start()
def test5():
s = multiprocessing.Semaphore(2)
for i in range(5):
p = multiprocessing.Process(target=worker, args=(s, i * 2))
p.start()
os.makedirs(output_folder)
mxd_paths = make_file_paths_list(input_folder, 'mxd', filename_pattern)
if keyword_filters:
mxd_paths = [
path for path in mxd_paths
if any(kw in path for kw in keyword_filters if kw)
]
logging.info(' Filtered files: {0}'.format(len(mxd_paths)))
print
total = len(mxd_paths)
jobs = []
sema = multiprocessing.Semaphore(cpu_limit)
logging.info('\n' + 'Exporting' + '\n')
start_time = datetime.datetime.now()
for i, path in enumerate(mxd_paths):
mxd_basename = os.path.basename(path)
output = os.path.join(
output_folder,
mxd_basename.replace('mxd', EXPORT_FUNCTIONS[output_type][1]))
if not overwrite:
if os.path.exists(output):
logging.warning('[ EXISTS ] {0} ({1} z {2})'.format(
mxd_basename, i + 1, total))
except Exception as e:
print(e)
print('------------response error')
r.put((-1, name))
semaphore.release()
if __name__ == '__main__':
getRank()
st = time()
iq = mp.Queue()
oq = mp.Queue()
mpnum = mp.Semaphore(pnum)
keyword = search
pid = 0
f = open('img/log.txt', 'w')
f.close()
keyword = search
print(keyword)
for key in keyword:
iq.put(key)
procs = []
l = len(keyword)
semE.acquire()
print("fin processus E")
semF.release()
def F(semE):
print("F: attend E et D")
semF.acquire()
print("F:1 jeton")
semF.acquire()
print("fin processus F")
semB = mp.Semaphore(0)
semC = mp.Semaphore(0)
semD = mp.Semaphore(0)
semE = mp.Semaphore(0)
semF = mp.Semaphore(0)
ProcessB = mp.Process(target=B, args=(semB, semE))
ProcessC = mp.Process(target=C, args=(semC, semE))
ProcessD = mp.Process(target=D, args=(semD, semF))
ProcessE = mp.Process(target=E, args=(semE, semF))
ProcessF = mp.Process(target=F, args=(semE, ))
ProcessB.start()
ProcessC.start()
ProcessD.start()
ProcessE.start()
def fastload(fname, firstn):
t0 = time.time()
f = h5py.File(fname, 'r')
assert ('X' in f.keys() and 'var' in f.keys() and 'obs' in f.keys())
# get obs dataframe
rows = f['obs'][list(f['obs'].keys())[0]].size
print("File has", rows, "rows")
if firstn > 1 and firstn < rows:
rows = firstn
print("Loading", rows, "rows")
if '_index' in f['obs'].keys():
dfobsind = pd.Series(f['obs']['_index'].asstr()[0:rows])
dfobs = pd.DataFrame(index=dfobsind)
else:
dfobs = pd.DataFrame()
for k in f['obs'].keys():
if k == '_index': continue
dfobs[k] = f['obs'][k].asstr()[...]
# get var dataframe
if '_index' in f['var'].keys():
dfvarind = pd.Series(f['var']['_index'].asstr()[...])
dfvar = pd.DataFrame(index=dfvarind)
else:
dfvar = pd.DataFrame()
for k in f['var'].keys():
if k == '_index': continue
dfvar[k] = f['var'][k].asstr()[...]
print("Pandas load done", time.time() - t0)
# load index pointers, prepare shared arrays
indptr = f['X']['indptr'][0:rows + 1]
datalen = int(indptr[-1])
if use_fastload == 1:
data = f['X']['data'][0:datalen]
indices = f['X']['indices'][0:datalen]
if use_fastload == 2:
f.close()
dataArray = mp.Array('f', datalen,
lock=False) # should be in shared memory
indicesArray = mp.Array(indices_shm_type, datalen,
lock=False) # should be in shared memory
print("created shared memory buffers", time.time() - t0)
#k = mkl.get_max_threads()
k = numba.get_num_threads()
procs = [
mp.Process(target=_load_helper,
args=(fname, i * datalen // k, (i + 1) * datalen // k,
dataArray, indicesArray)) for i in range(k)
]
for p in procs:
p.start()
for p in procs:
p.join()
print("multiproc load done", time.time() - t0)
data = np.frombuffer(dataArray, dtype=np.float32)
indices = np.frombuffer(indicesArray, dtype=indices_type)
if use_fastload == 3:
f.close()
#k = mkl.get_max_threads()
k = numba.get_num_threads()
dataArray = mp.Array('f', k * 1024 * 1024,
lock=False) # should be in shared memory
indicesArray = mp.Array(indices_shm_type, k * 1024 * 1024,
lock=False) # should be in shared memory
startsArray = mp.Array('l', k, lock=False) # start index of data read
endsArray = mp.Array(
'l', k, lock=False) # end index (noninclusive) of data read
barrier = mp.Barrier(k + 1) # k workers plus this process
dataA = np.frombuffer(dataArray, dtype=np.float32)
indicesA = np.frombuffer(indicesArray, dtype=indices_type)
startsA = np.frombuffer(startsArray, dtype=np.int64)
endsA = np.frombuffer(endsArray, dtype=np.int64)
data = np.empty(datalen, dtype=np.float32)
indices = np.empty(datalen, dtype=indices_type)
print("created shared memory buffers", time.time() - t0)
procs = [
mp.Process(target=_load_helper2,
args=(fname, i, k, datalen, barrier, dataArray,
indicesArray, startsArray, endsArray))
for i in range(k)
]
for p in procs:
p.start()
for _ in range(datalen // (k * 1024 * 1024) + 1):
barrier.wait() # wait for data loads
_fast_copy(data, dataA, startsA, endsA, k)
_fast_copy(indices, indicesA, startsA, endsA, k)
barrier.wait() # finish data copy
for p in procs:
p.join()
print("multiproc load done", time.time() - t0)
if use_fastload == 4 or use_fastload == 5:
f.close()
#k = mkl.get_max_threads()
k = numba.get_num_threads()
dataArray = mp.Array('f', k * 1024 * 1024,
lock=False) # should be in shared memory
indicesArray = mp.Array(indices_shm_type, k * 1024 * 1024,
lock=False) # should be in shared memory
startsArray = mp.Array('l', k, lock=False) # start index of data read
endsArray = mp.Array(
'l', k, lock=False) # end index (noninclusive) of data read
global semDataLoaded
global semDataCopied
semDataLoaded = [mp.Semaphore(0) for _ in range(k)]
semDataCopied = [mp.Semaphore(0) for _ in range(k)]
dataA = np.frombuffer(dataArray, dtype=np.float32)
indicesA = np.frombuffer(indicesArray, dtype=indices_type)
startsA = np.frombuffer(startsArray, dtype=np.int64)
endsA = np.frombuffer(endsArray, dtype=np.int64)
data = np.empty(datalen, dtype=np.float32)
indices = np.empty(datalen, dtype=indices_type)
print("created shared memory buffers", time.time() - t0)
procs = [
mp.Process(target=_load_helper3,
args=(fname, i, k, datalen, dataArray, indicesArray,
startsArray, endsArray)) for i in range(k)
]
for p in procs:
p.start()
if use_fastload == 4:
threads = [
threading.Thread(target=_copy_thread,
args=(data, dataA, indices, indicesA, startsA,
endsA, i,
datalen // (k * 1024 * 1024) + 1))
for i in range(k)
]
for t in threads:
t.start()
else:
_fast_copy3(data, dataA, indices, indicesA, startsA, endsA, k,
datalen // (k * 1024 * 1024) + 1)
for p in procs:
p.join()
if use_fastload == 4:
for t in threads:
t.join()
print("multiproc load done", time.time() - t0)
#X = scipy.sparse.csr_matrix((data,indices,indptr)) #This is too slow
X = scipy.sparse.csr_matrix((0, 0))
#X = scipy.sparse.csr_matrix((rows,dfvar.shape[0]))
X.data = data
X.indices = indices
#X.data = np.empty(data.shape,dtype=data.dtype)
#X.data[:] = data
#X.indices = np.empty(indices.shape,dtype=indices.dtype)
#X.indices[:] = indices
X.indptr = indptr
X._shape = ((rows, dfvar.shape[0]))
print("Created scipy sparse array", time.time() - t0)
# create AnnData
adata = anndata.AnnData(X, dfobs, dfvar)
print("anndata created", time.time() - t0)
return adata
if len(sys.argv) != 4:
raise ValueError(
'\n\nSyntax must be:\n\t python multiprocexec -n N_proc sim_class=module.class\n\n'
)
if '-n' not in sys.argv[1] or 'sim_class' not in sys.argv[3]:
raise ValueError(
'\n\nSyntax must be:\n python multiprocexec -n N_proc sim_class=module.class\n\n'
)
N_proc = int(sys.argv[2])
sim_module_string = sys.argv[3].split('=')[-1]
queue_list = [mp.Queue() for _ in xrange(N_proc)]
mutex = mp.Semaphore(1)
barrier = mp.Semaphore(0)
turnstile = mp.Semaphore(0)
turnstile2 = mp.Semaphore(1)
cnt = mp.Value('i', 0)
proc_list = []
for pid in xrange(N_proc):
proc_list.append(
mp.Process(target=todo,
args=(sim_module_string, pid, N_proc, queue_list, mutex,
barrier, turnstile, turnstile2, cnt)))
for p in proc_list:
p.start()
for p in proc_list:
p.join()
def sem_test():
s = multiprocessing.Semaphore(1)
for i in range(5):
p = multiprocessing.Process(target=sem_worker, args=(s, i * 2))
p.start()
import sys
import os
import requests
import time
try:
from urllib.parse import urlparse
except ImportError:
from urlparse import urlparse
from nhentai.logger import logger
from nhentai.parser import request
from nhentai.utils import Singleton
requests.packages.urllib3.disable_warnings()
semaphore = multiprocessing.Semaphore(1)
class NHentaiImageNotExistException(Exception):
pass
class Downloader(Singleton):
def __init__(self, path='', size=5, timeout=30, delay=0):
self.size = size
self.path = str(path)
self.timeout = timeout
self.delay = delay
def download_(self, url, folder='', filename='', retried=0):
if self.delay:
def predict(
data=None,
predicted_column: None | str | int = None,
config: predictit.configuration.Config | dict | None = None,
**kwargs,
) -> predictit._result_classes.Result:
"""Make predictions mostly on time-series data. Data input and other config options can be set up in
configuration.py or overwritten on the fly. Setup can be also done as function input arguments or as command line
arguments (it will overwrite config values).
There are working examples in main readme and also in test_it module.
Function can be configured from with config from configuration, with command line arguments as wel as with
function parameters. There are only two possible positional parameters - `data` and `predicted_column`. Rest of
parameters must be named parameters. Params are not documented here, because all config params works here in
function passed as kwargs.
Args:
data (np.ndarray, pd.DataFrame, str): Time series. Can be 2-D - more columns. Can be numpy array, DataFrame,
path to file or url.
Examples: "/home/user/my.json", or "https://yoururl/your.csv" or np.random.randn(100, 2).
predicted_column (None | str | int, optional): Index of predicted column or it's name (dataframe).
If list with more values only the first one will be evaluated (use predict_multiple_columns function
if you need that. Default to None.
config (predictit.configuration.Config | dict | None, optional): Settings as Config instance or dictionary.
Check class for what you can use. If None, then default config will be used. Defaults to None.
**kwargs (dict, optional): There is much more parameters' of predict function. Check configuration.py
for parameters details.
Returns:
Depend on 'return_type' config value - return best prediction {np.ndarray}, all models results {np.ndarray},
detailed results{dict} or interactive plot or print tables of results
"""
from mypythontools.plots import plot
if config is None or isinstance(config, dict):
update_config = config
config = config_default
config = config.copy()
if update_config:
config.update(update_config)
elif isinstance(config, predictit.configuration.Config):
config = config.copy()
if config.use_config_preset and config.use_config_preset != "none":
updated_config = config.presets[config.use_config_preset]
config.update(updated_config)
# Edit configuration.py default values with arguments values if exist
if data is not None:
config.data = data
if predicted_column is not None:
config.predicted_column = predicted_column
config.update(kwargs)
predictit._helpers.logger_init_from_config(config.output.logger_subconfig)
# Do not repeat actually mean evaluate once
if not config.repeatit:
config.repeatit = 1
_GUI = GLOBAL_VARS.GUI
# Add everything printed + warnings to variable to be able to print in GUI
if _GUI:
stdout = sys.stdout
sys.stdout = io.StringIO()
# Don't want to define in gui condition, so if not gui, do nothing
if _GUI:
def update_gui(content, html_id):
try:
predictit.gui_start.edit_gui_py(content, html_id)
except (Exception, ):
pass
else:
def update_gui(content, html_id):
pass
# Definition of the table for spent time on code parts
time_df = []
def update_time_table(time_last):
time_df.append([progress_phase, round((time.time() - time_last), 3)])
return time.time()
time_point = time_begin = time.time()
###############
### ANCHOR ### Data
#############
progress_phase = "Data loading and preprocessing"
update_gui(progress_phase, "progress_phase")
data = mdp.load_data.load_data(
config.data,
header=config.header,
csv_style=config.csv_style,
predicted_table=config.predicted_table,
max_imported_length=config.max_imported_length,
request_datatype_suffix=config.request_datatype_suffix,
data_orientation=config.data_orientation,
)
###############
### ANCHOR ### Data consolidation
#############
if not config.predicted_column:
config.predicted_column = 0
data_for_predictions_df = mdp.preprocessing.data_consolidation(
data,
predicted_column=config.predicted_column,
other_columns=config.other_columns,
datalength=config.datalength,
datetime_column=config.datetime_column,
unique_threshold=config.unique_threshold,
embedding=config.embedding,
freq=config.freq,
resample_function=config.resample_function,
remove_nans_threshold=config.remove_nans_threshold,
remove_nans_or_replace=config.remove_nans_or_replace,
dtype=config.dtype,
)
# In data consolidation predicted column was replaced on index 0 as first column
predicted_column_index = 0
predicted_column_name = data_for_predictions_df.columns[0]
###############
### ANCHOR ### Analyze original data
#############
column_for_predictions_series = data_for_predictions_df.iloc[:, 0:1]
results = {}
data_inputs = []
if config.mode == "validate":
column_for_predictions_series = column_for_predictions_series.iloc[:
-config
.
output
.
predicts, :]
config.repeatit = 1
for i in config.used_models:
data_inputs.append(config.models_input[i])
data_inputs = set(data_inputs)
if config.analyzeit == 1 or config.analyzeit == 3:
print("Analyze of unprocessed data")
try:
predictit.analyze.analyze_column(data_for_predictions_df.values[:,
0],
window=30)
predictit.analyze.analyze_data(data_for_predictions_df)
predictit.analyze.decompose(
data_for_predictions_df.values[:, 0],
**config.analyze_seasonal_decompose,
)
except Exception:
mylogging.traceback("Analyze failed", level="ERROR")
semaphor = None
if config.multiprocessing:
multiprocessing.freeze_support()
if not config.processes_limit:
config.processes_limit = multiprocessing.cpu_count() - 1
if config.multiprocessing == "process":
pipes = []
semaphor = multiprocessing.Semaphore(config.processes_limit)
elif config.multiprocessing == "pool":
pool = multiprocessing.Pool(config.processes_limit)
# It is not possible easy share data in multiprocessing, so results are resulted via callback function
def return_result(result):
for i, j in result.items():
results[i] = j
### Optimization loop
if (not config.optimization or not config.optimization_variable
or not config.optimization_values
or len(config.optimization_values) == 1):
config.variable_optimization.optimization = False
config.optimization_values = ["Not optimized"]
config.optimization_variable = None
time_point = update_time_table(time_point)
progress_phase = "Predict"
update_gui(progress_phase, "progress_phase")
models_indexed = {i: j for i, j in enumerate(config.used_models)}
###############
### ANCHOR ### Main loop
#############
for optimization_index, optimization_value in enumerate(
config.optimization_values):
# TODO check why setattr - may be wrong after config change
if config.optimization_variable:
setattr(config, config.optimization_variable, optimization_value)
###############
### ANCHOR ### Feature extraction
#############
if config.add_fft_columns:
data_for_predictions_df = mdp.feature_engineering.add_frequency_columns(
data_for_predictions_df,
window=config.feature_engineering.add_fft_columns,
)
if config.data_extension:
data_for_predictions_df = mdp.feature_engineering.add_derived_columns(
data_for_predictions_df,
**config.feature_engineering.data_extension)
###############
### ANCHOR ### Feature selection
#############
# data_for_predictions_df TODO
###############
### ANCHOR ### Data preprocessing
#############
if config.mode == "validate":
test_unstandardized = mdp.misc.split(
data_for_predictions_df, predicts=config.predicts)[1].values
models_test_outputs_unstandardized = [test_unstandardized]
else:
models_test_outputs_unstandardized = mdp.create_model_inputs.create_tests_outputs(
data_for_predictions_df.values[:, 0],
predicts=config.predicts,
repeatit=config.repeatit,
)
data_for_predictions, last_undiff_value, final_scaler = mdp.preprocessing.preprocess_data(
data_for_predictions_df.values,
remove_outliers=config.remove_outliers,
smoothit=config.smoothit,
correlation_threshold=config.correlation_threshold,
data_transform=config.data_transform,
standardizeit=config.standardizeit,
bins=config.bins,
binning_type=config.binning_type,
)
data_for_predictions = cast(np.ndarray, data_for_predictions)
if config.mode == "validate":
data_for_predictions, test = mdp.misc.split(
data_for_predictions, predicts=config.predicts)
models_test_outputs = [test]
else:
models_test_outputs = mdp.create_model_inputs.create_tests_outputs(
data_for_predictions[:, 0],
predicts=config.predicts,
repeatit=config.repeatit,
)
column_for_predictions_processed = data_for_predictions[:,
predicted_column_index]
data_shape = np.shape(data_for_predictions)
data_length = len(column_for_predictions_processed)
data_std = np.std(column_for_predictions_processed[-30:])
data_mean = np.mean(column_for_predictions_processed[-30:])
data_abs_max = max(
abs(column_for_predictions_processed.min()),
abs(column_for_predictions_processed.max()),
)
multicolumn = 0 if data_shape[1] == 1 else 1
if (config.analyzeit == 2 or config.analyzeit == 3
) and optimization_index == len(config.optimization_values) - 1:
print("\n\nAnalyze of preprocessed data\n")
try:
predictit.analyze.analyze_column(
column_for_predictions_processed, window=30)
predictit.analyze.analyze_data(data_for_predictions)
predictit.analyze.decompose(
column_for_predictions_processed,
**config.analyze_seasonal_decompose,
)
except Exception:
mylogging.traceback("Analyze failed", level="ERROR")
min_data_length = 3 * config.predicts + config.default_n_steps_in
if (data_length < min_data_length or data_length <
config.repeatit + config.default_n_steps_in + config.predicts):
config.repeatit = 1
min_data_length = 3 * config.predicts + config.default_n_steps_in
assert min_data_length < data_length, mylogging.return_str(
"Set up less predicted values in settings or add more data",
caption="To few data",
)
for data_inputs_name in data_inputs:
try:
(
model_train_input,
model_predict_input,
model_test_inputs,
) = mdp.create_model_inputs.create_inputs(
data_for_predictions,
input_type_name=data_inputs_name,
input_type_params=config.data_inputs[data_inputs_name],
mode=config.mode,
predicts=config.predicts,
repeatit=config.repeatit,
predicted_column_index=predicted_column_index,
)
except Exception:
mylogging.traceback(
f"Error in creating input type: {data_inputs_name} with option optimization: {optimization_value}",
level="WARNING",
)
continue
for (iterated_model_index,
iterated_model_name) in models_indexed.items():
iterated_model = predictit.models.models_assignment[
iterated_model_name]
if config.models_input[
iterated_model_name] == data_inputs_name:
predict_parameters = {
"config": config.get_dict(),
# Functions to not import all modules
"preprocess_data_inverse":
mdp.preprocessing.preprocess_data_inverse,
"fitted_power_transform":
mdp.preprocessing.fitted_power_transform,
# Other
"iterated_model_train": iterated_model.train,
"iterated_model_predict": iterated_model.predict,
"iterated_model_name": iterated_model_name,
"iterated_model_index": iterated_model_index,
"optimization_index": optimization_index,
"optimization_value": optimization_value,
"model_train_input": model_train_input,
"model_predict_input": model_predict_input,
"model_test_inputs": model_test_inputs,
"models_test_outputs": models_test_outputs,
"models_test_outputs_unstandardized":
models_test_outputs_unstandardized,
"data_abs_max": data_abs_max,
"data_mean": data_mean,
"data_std": data_std,
"last_undiff_value": last_undiff_value,
"final_scaler": final_scaler,
"semaphor": semaphor,
}
if config.models_input[iterated_model_name] in [
"one_step",
"one_step_constant",
]:
if multicolumn and config.predicts > 1:
mylogging.warn(
f"Warning in model {iterated_model_name} \n\nOne-step prediction on "
"multivariate data (more columns). Use multi_step (y lengt equals to predict) "
"or do use some one column data input in config models_input or predict just one value."
)
continue
if config.multiprocessing == "process":
pipes.append(multiprocessing.Pipe(duplex=False))
p = multiprocessing.Process(
target=predictit._main_loop.train_and_predict,
kwargs={
**predict_parameters,
**{
"pipe": pipes[-1][1]
}
},
)
p.Daemon = True # Baby process will be terminated if parent killed
p.start()
elif config.multiprocessing == "pool":
pool.apply_async(
predictit._main_loop.train_and_predict,
(),
predict_parameters,
callback=return_result,
)
else:
results = {
**results,
**predictit._main_loop.train_and_predict(**predict_parameters),
}
if config.multiprocessing:
if config.multiprocessing == "process":
for i in pipes:
try:
results = {**results, **i[0].recv()}
except Exception:
pass
if config.multiprocessing == "pool":
pool.close()
pool.join()
for i in results.values():
mylogging.my_logger.log_and_warn_from_lists(
i["logs_list"], i["warnings_list"])
# Create confidence intervals
if config.confidence_interval:
try:
lower_bound, upper_bound = predictit.misc.confidence_interval(
column_for_predictions_series.values,
predicts=config.predicts,
confidence=config.confidence_interval,
)
grey_area = ["Lower bound", "Upper bound"]
bounds = True
except Exception:
bounds = False
grey_area = ["Lower bound", "Upper bound"]
mylogging.traceback("Error in compute confidence interval",
level="ERROR")
else:
bounds = False
grey_area = False
###############
### ANCHOR ### Results processing
#############
# Criterion is the best of average from repetitions
time_point = update_time_table(time_point)
progress_phase = "Evaluation"
update_gui(progress_phase, "progress_phase")
# Two kind of results we will create. Both as dataframe
# - First are all the details around prediction. Model errors, time, memory peak etc.
# - Second we have predicted values
# Results such as trained model etc. that cannot be displayed in dataframe are in original results dict.
# Convert results from dictionary to dataframe - exclude objects like trained model
results_df = pd.DataFrame.from_dict(results, orient="index")
if results_df.empty:
raise RuntimeError(
mylogging.return_str(
"None of models finished predictions. Set config.logger_level = 'DEBUG' for more info.",
caption="All models failed for some reason",
))
evaluated_matrix = np.zeros(
(1, len(config.optimization_values), len(config.used_models)))
evaluated_matrix.fill(np.nan)
for k in results.values():
evaluated_matrix[0, k["Index"][0], k["Index"][1]] = k["Model error"]
(
_,
best_models_optimized_values,
optimized_values_results_df,
best_model_name,
best_optimized_value,
) = predictit.analyze.analyze_results(
evaluated_matrix,
config.optimization_values,
config.models.used_models,
config.prediction.error_criterion,
)
# Generate date indexes for result predictions
last_date = column_for_predictions_series.index[-1]
if isinstance(
last_date,
(pd.core.indexes.datetimes.DatetimeIndex,
pd._libs.tslibs.timestamps.Timestamp),
):
date_index = pd.date_range(
start=last_date,
periods=config.predicts + 1,
freq=column_for_predictions_series.index.freq,
)[1:]
date_index = pd.to_datetime(date_index)
else:
date_index = list(range(last_date + 1,
last_date + config.predicts + 1))
predictions_df = pd.DataFrame(index=date_index)
results_df.sort_values("Model error", inplace=True)
for i, row in results_df.iterrows():
predictions_df[i] = row["Results"]
if predictions_df.empty:
raise RuntimeError(
mylogging.return_str(
"Neither of models finished prediction. Set config.logger_level = 'DEBUG' for more info."
))
if config.variable_optimization.optimization:
best_optimized_values_dict = {
j: best_models_optimized_values[i]
for i, j in enumerate(config.used_models)
}
best_indexes = []
for i, row in results_df.iterrows():
if row["Optimization value"] == best_optimized_values_dict[
row["Name"]]:
best_indexes.append(i)
optimization_result = predictit._result_classes.Optimization(
optimized_variable=config.variable_optimization.
optimization_variable,
optimized_options=config.variable_optimization.optimization_values,
best_value=best_optimized_value,
values_results_df=optimized_values_results_df,
best_values_for_models=best_optimized_values_dict,
all_models_results_df=results_df,
all_models_predictions_df=predictions_df,
)
predictions_df = predictions_df[best_indexes]
predictions_df.columns = [
results_df.loc[i]["Name"] for i in predictions_df.columns
]
results_df = results_df.loc[best_indexes]
results_df.rename(columns={"A": "Col_1"}, inplace=True)
else:
optimization_result = None
if config.hyperparameter_optimization.optimizeit:
hyperparameter_optimization_kwargs = results_df[
"Best optimized parameters"].to_dict()
else:
hyperparameter_optimization_kwargs = None
results_df.set_index("Name", inplace=True)
results_to_drop = [
i for i in [
"Index",
"Trained model",
"Test errors",
"Results",
"logs_list",
"warnings_list",
] if i in results_df.columns
]
results_df.drop(columns=results_to_drop, inplace=True)
best_model_predicts = predictions_df[best_model_name]
###############
### ANCHOR ### Plot
#############
if config.variable_optimization.optimization and config.variable_optimization.plot_all_optimized_models:
predictions_for_plot = optimization_result.all_models_predictions_df.copy(
)
else:
predictions_for_plot = predictions_df.copy()
predictions_for_plot.columns = [
f"{i + 1} - {j}" for i, j in enumerate(predictions_for_plot.columns)
]
if config.mode == "validate":
best_model_name_plot = "Test"
predictions_df.insert(0, "Test", test_unstandardized)
predictions_for_plot.insert(0, "Test", test_unstandardized)
else:
best_model_name_plot = predictions_for_plot.columns[0]
bounds_df = pd.DataFrame(index=date_index)
if bounds:
bounds_df["Upper bound"] = upper_bound
bounds_df["Lower bound"] = lower_bound
last_value = float(column_for_predictions_series.iloc[-1, 0])
predictions_for_plot_limited = pd.concat(
[
predictions_for_plot.iloc[:, :config.plot_number_of_models],
bounds_df
],
axis=1,
)
predictions_with_history = pd.concat(
[
column_for_predictions_series[-config.plot_history_length:],
predictions_for_plot_limited,
],
sort=False,
)
predictions_with_history.iloc[-config.predicts - 1, :] = last_value
if config.sort_results_by == "name":
results_df.sort_index(key=lambda x: x.str.lower(), inplace=True)
predictions_df.sort_index(key=lambda x: x.str.lower(), inplace=True)
if config.general.analyzeit:
import matplotlib.pyplot as plt
plt.show()
time_point = update_time_table(time_point)
progress_phase = "plot"
update_gui(progress_phase, "progress_phase")
if config.output.plot_subconfig.show_plot or config.output.plot_subconfig.save_plot:
with warnings.catch_warnings():
warnings.simplefilter("ignore", ResourceWarning)
return_div = True if _GUI else False
if config.plot_type == "with_history":
div = plot(
predictions_with_history,
plot_library=config.plot_library,
plot_name=config.plot_name,
legend=config.plot_legend,
highlighted_column=predicted_column_name,
surrounded_column=best_model_name_plot,
grey_area=grey_area,
save=config.save_plot,
return_div=return_div,
show=config.output.plot_subconfig.show_plot,
)
elif config.plot_type == "just_results":
div = plot(
predictions_for_plot,
plot_library=config.plot_library,
legend=config.plot_legend,
highlighted_column=best_model_name_plot,
save=config.save_plot,
show=config.output.plot_subconfig.show_plot,
)
update_time_table(time_point)
progress_phase = "Completed"
update_gui(progress_phase, "progress_phase")
###############
### ANCHOR ### Table
#############
time_df.append(["Complete time", round((time.time() - time_begin), 3)])
time_df = pd.DataFrame(time_df, columns=["Part", "Time"])
simple_table_df = mdp.misc.edit_table_to_printable(results_df[[
"Model error"
]].iloc[:config.print_number_of_models, :].reset_index())
detailed_table_df = results_df.iloc[:config.
print_number_of_models, :].reset_index(
)
detailed_table_df.drop(["Unstandardized model error"],
axis=1,
inplace=True)
detailed_table_df = mdp.misc.edit_table_to_printable(detailed_table_df)
tables = predictit._result_classes.Tables(
simple=tabulate(
simple_table_df.values,
headers=["Model", f"Average {config.error_criterion} error"],
**config.table_settings,
),
detailed=tabulate(
detailed_table_df.values,
headers=detailed_table_df.columns,
**config.table_settings,
),
time=tabulate(time_df.values,
headers=time_df.columns,
**config.table_settings),
simple_table_df=simple_table_df,
detailed_table_df=detailed_table_df,
)
###############
### ANCHOR ### Results
#############
misc_result = predictit._result_classes.Misc(
evaluated_matrix=evaluated_matrix)
result = predictit._result_classes.Result(
best_prediction=best_model_predicts,
best_model_name=best_model_name,
predictions=predictions_df,
results_df=results_df,
results=results,
with_history=predictions_with_history,
tables=tables,
config=config,
misc=misc_result,
optimization=optimization_result,
hyperparameter_optimization_kwargs=hyperparameter_optimization_kwargs,
)
###############
### ANCHOR ### Print
#############
if config.print_result_details:
print((
f"\nBest model is {best_model_name} with results \n\n{best_model_predicts}\n\nWith model error {config.error_criterion} = "
f"{results_df.loc[best_model_name, 'Model error']}"))
if config.print_table == "simple":
print(f"\n{tables.simple}\n")
elif config.print_table == "detailed":
print(f"\n{tables.detailed}\n")
if config.print_time_table:
print(f"\n{tables.time}\n")
###############
### ANCHOR ### Return
#############
mylogging.reset_outer_warnings_filter()
# Return stdout and stop collect warnings and printed output
if _GUI:
output = sys.stdout.getvalue()
sys.stdout = stdout
result.output = output
print(output)
if _GUI:
result.plot = div
if config.return_internal_results:
return {
"data_for_predictions (X, y)": data_for_predictions,
"model_train_input": model_train_input,
"model_predict_input": model_predict_input,
"model_test_inputs": model_test_inputs,
"models_test_outputs": models_test_outputs,
}
return result
if (count == 20):
time_end = time.clock()
print("消费 " + str(count) + "个商品所用时间: %f s" %
(time_end - time_start))
mutex.release()
empty.release()
time.sleep(1)
if __name__ == '__main__':
product = multiprocessing.Array('d', range(5))
pindex = multiprocessing.Value('i', 0)
cindex = multiprocessing.Value('i', 0)
mutex = multiprocessing.Semaphore(1)
full = multiprocessing.Semaphore(0)
empty = multiprocessing.Semaphore(5)
Process(target=produce,
args=(1, mutex, full, empty, product, pindex, cindex)).start()
Process(target=produce,
args=(2, mutex, full, empty, product, pindex, cindex)).start()
Process(target=consume,
args=(mutex, full, empty, product, pindex, cindex)).start()
csv_bands.close()
if __name__ == '__main__':
print("Mapping products to csv...")
n_processes = config.n_processes_generate_dataset
product_directory = paths.product_directory
tmp = paths.tmp
if not(exists(tmp)):
mkdir(tmp)
if not(exists(paths.dataset_directory)):
mkdir(paths.dataset_directory)
products= listdir(product_directory)
semaphore = multiprocessing.Semaphore(n_processes)
progress_bar = Progress_bar(len(products))
job_list = []
for product in products:
semaphore.acquire()
job = multiprocessing.Process(target=process, args=(product,semaphore,progress_bar, tmp))
job_list.append(job)
job.start()
for j in job_list:
j.join()
csv_index = open(paths.csv_index, 'wb+')
csv_bands = open(paths.csv_band, 'wb+')
csv_index.write(b'DATE;LONGITUDE;LATITUDE;NDVI;SLAVI;GVMI;NDWI;BSI;NPCRI;CLASS\n')
csv_bands.write(b'DATE;LONGITUDE;LATITUDE;B01;B02;B03;B04;B05;B06;B07;B08;B8A;B09;B11;B12;CLASS\n')
# Obtain possible configuration from the environment, assuming 1 (on)
# by default, upon 0 set to None. Should instructively fail if some non
# 0/1 value is set.
multiprocessing = int(os.environ.get('JOBLIB_MULTIPROCESSING', 1)) or None
if multiprocessing:
try:
import multiprocessing
except ImportError:
multiprocessing = None
# 2nd stage: validate that locking is available on the system and
# issue a warning if not
if multiprocessing:
try:
_sem = multiprocessing.Semaphore()
del _sem # cleanup
except (ImportError, OSError) as e:
multiprocessing = None
warnings.warn('%s. joblib will operate in serial mode' % (e, ))
from .format_stack import format_exc, format_outer_frames
from .logger import Logger, short_format_time
from .my_exceptions import TransportableException, _mk_exception
###############################################################################
# CPU that works also when multiprocessing is not installed (python2.5)
def cpu_count():
""" Return the number of CPUs.
"""
import serial, time
import threading, random
import multiprocessing as mp
import requests, json
# CONSTANT
SERIALPORT = "COM7"
BAUDRATE = 115200
API_SENSOR_URL = "http://localhost:8081/api/sensors"
# GLOBAL
nValue, nEndTransmission = 0, 0
locationX, locationY = 0, 0
ser = serial.Serial()
semaphoreSendingData = mp.Semaphore(1)
def getFireHTTP(sLocationX, sLocationY):
headers = {"Content-Type": "application/json"}
result = requests.get(url=API_SENSOR_URL, headers=headers)
jsonValue = json.loads(result.content.decode("utf-8"))
for i in range(len(jsonValue)):
if (jsonValue[i])["x"] == int(sLocationX) and (
jsonValue[i])["y"] == int(sLocationY):
return ((jsonValue[i])["intensity"])
# Function in charge of initialize the serial port
def initUART():
ser.port = SERIALPORT
ser.baudrate = BAUDRATE
ser.bytesize = serial.EIGHTBITS #number of bits per bytes
# -*- coding: utf-8 -*-
"""
Created on Fri Jun 1 10:35:45 2018
@author: antoine.bralet
"""
import multiprocessing as mp
import sys
import os
import time
verrou = mp.Semaphore(1)
verrou2 = mp.Semaphore(0)
verrou3 = mp.Semaphore(0)
def A():
verrou.acquire()
print("Vive le dodo de A")
time.sleep(2)
print("Coucou je suis le processus A et j'ai attendu 2 secondes")
verrou.release()
verrou.release()
verrou.release()
def B():
verrou.acquire()
print("Vive le dodo de B")
time.sleep(3)
print("Coucou je suis le processus B et j'ai attendu 3 secondes")
# -*- coding: utf-8 -*-
"""
Created on Fri Jun 1 11:53:36 2018
@author: antoine.bralet
"""
import multiprocessing as mp
import sys
import os
import time
import random
patates = mp.Semaphore(0)
fromage = mp.Semaphore(0)
menage = mp.Semaphore(0)
def rdv1():
print("Je suis la ca y est")
fromage.release()
menage.release()
patates.acquire()
patates.acquire()
print("Il faut que j'aille chercher des patates pour la raclette")
time.sleep(4)
def rdv2():
print("Je suis la ca y est")
patates.release()
menage.release()
fromage.acquire()