def checkAllDatasets(self):
"""
Look for corrupted files in the whole catalog.
"""
catalog = self.readCatalog()
self.parallel_ = Parallel(50,self.queue_)
## self.parallel_ = Parallel(1,self.queue_)
print "Checking all datasets"
for dataset in catalog.keys():
self.checkDatasetFiles(dataset,catalog)
outcomes = self.parallel_.wait(printOutput=False)
## for dsetName,ifile,fName,ret,out in outcomes:
for ign1, ign2, outcome in outcomes:
dsetName,ifile,fName,ret,out = outcome
info = catalog[dsetName]["files"][ifile]
if info["name"] != fName:
print "Inconsistent outcome ", info["name"], dsetName,ifile,fName,ret,out
else:
if ret != 0:
info["bad"] = True
else:
extraInfo = json.loads(str(out))
for key,val in extraInfo.iteritems():
info[key] = val
print "Writing catalog"
self.writeCatalog(catalog)
print "Done"
class USARTTest(unittest.TestCase):
par = Parallel()
def __init__(self, *args):
unittest.TestCase.__init__(self, *args)
self.ser = Serial(_SERIAL_PATH,
baudrate=_BAUDRATE,
bytesize=8,
parity='N',
stopbits=1,
timeout=1,
xonxoff=0,
rtscts=0)
self._toWrite = "Write_TEST123"
self._toRead = "Read_TEST123"
def testread(self):
time.sleep(0.1)
USARTTest.par.setData(0x01)
time.sleep(0.1)
self.ser.write(self._toRead + '\n')
USARTTest.par.setData(0x00)
print "See avr console..."
def testwrite(self):
time.sleep(0.1)
USARTTest.par.setData(0x01)
line = self.ser.readline(len(self._toWrite) + 1, '\n')
USARTTest.par.setData(0x00)
self.assertNotEqual(len(line), 0)
self.assertNotEqual(line.find(self._toWrite), -1, "Received %s" % line)
def __init__(self,
mode='dummy',
address=None,
high_duration=0.005,
verbose=None):
if mode == 'parallel':
if sys.platform.startswith('linux'):
address = '/dev/parport0' if address is None else address
if not isinstance(address, string_types):
raise ValueError('addrss must be a string or None, got %s '
'of type %s' % (address, type(address)))
from parallel import Parallel
self._port = Parallel(address)
self._portname = address
self._set_data = self._port.setData
elif sys.platform.startswith('win'):
from ctypes import windll
if not hasattr(windll, 'inpout32'):
raise SystemError(
'Must have inpout32 installed, see:\n\n'
'http://www.highrez.co.uk/downloads/inpout32/')
base = 0x378 if address is None else address
if isinstance(base, string_types):
base = int(base, 16)
if not isinstance(base, int):
raise ValueError('address must be int or None, got %s of '
'type %s' % (base, type(base)))
self._port = windll.inpout32
mask = np.uint8(1 << 5 | 1 << 6 | 1 << 7)
# Use ECP to put the port into byte mode
val = int((self._port.Inp32(base + 0x402) & ~mask) | (1 << 5))
self._port.Out32(base + 0x402, val)
# Now to make sure the port is in output mode we need to make
# sure that bit 5 of the control register is not set
val = int(self._port.Inp32(base + 2) & ~np.uint8(1 << 5))
self._port.Out32(base + 2, val)
self._set_data = lambda data: self._port.Out32(base, data)
self._portname = str(base)
else:
raise NotImplementedError('Parallel port triggering only '
'supported on Linux and Windows')
else: # mode == 'dummy':
self._port = self._portname = None
self._trigger_list = list()
self._set_data = lambda x: (self._trigger_list.append(x)
if x != 0 else None)
self.high_duration = high_duration
self.mode = mode
def __init__(self, motor_inputs, state=0, delay=0.05):
'''
:param motor_inputs: Ordered list of parallel values to turn motor
:type motor_inputs: list or tuple
:param state: Initial starting state of motor position
:type state: int
:param delay: Delay between steps (speed)
:type delay: float
'''
self.MOTOR_INPUTS = motor_inputs
self.state = state
self.delay = delay
# Setup parallel interface on first init
self.parallel_interface = Parallel()
def __call__(self):
"""
__call__
Run all jobs.
"""
self.parallel = Parallel(self.options.ncpu,
lsfQueue=self.options.queue,
lsfJobName="%s/runJobs" %
self.options.outputDir,
asyncLsf=False)
self.jobs = None
if self.options.cont:
pass
else:
self.firstRun()
self.monitor()
def getFilesFomEOS(self,dsetName):
"""
Read dataset files crawling EOS.
@dsetName: dataset name
Note: not implemented
"""
if not self.parallel_:
self.parallel_ = Parallel(200,self.queue_)
ret,out = self.parallel_.run("/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select",["find",dsetName],interactive=True)[2]
## print out
files = []
for line in out.split("\n"):
if line.endswith(".root"):
files.append( {"name":line.replace("/eos/cms",""), "nevents":0} )
return files
def monitor(self):
(options, args) = (self.options, self.args)
parallel = self.parallel
with open("%s/task_config.json" % (options.outputDir), "r") as cfin:
task_config = json.loads(cfin.read())
doutfiles = task_config["datasets_output"]
poutfiles = task_config["process_output"]
outfiles = task_config["output"]
outputPfx = task_config["outputPfx"]
if not options.dry_run:
## FIXME: job resubmission
self.jobs = task_config["jobs"]
returns = self.wait(parallel, self)
task_config["jobs"] = self.jobs
if options.hadd:
print "All jobs finished. Merging output."
p = Parallel(options.ncpu)
hadd = "hadd -f "
if options.hadd_process:
for proc, out in poutfiles.iteritems():
outfile, outfiles = out
p.run("%s %s" % (hadd, outfile), outfiles)
if options.hadd_dataset:
if options.hadd_process:
hadd += " -T"
for dset, out in doutfiles.iteritems():
outfile, outfiles = out
p.run("%s %s" % (hadd, outfile), outfiles)
if not (options.hadd_process or options.hadd_dataset):
p.run("%s %s.root" % (hadd, outputPfx), outfiles)
self.wait(p)
with open("%s/task_config.json" % (options.outputDir), "w+") as cfout:
cfout.write(json.dumps(task_config, indent=4))
cfout.close()
self.parallel.stop()
def insertMarks(expInfo, nombreEDF):
ponermarcas = []
if expInfo[EXPERIMENT_TYPE] == EMOTIV:
from multiprocessing import Process, Queue
import guardar
q_marcas = Queue()
p = Process(target=guardar.save_data, args=(
nombreEDF,
q_marcas,
))
p.start()
ponermarcas = 1
elif expInfo[EXPERIMENT_TYPE] == TRADITIONAL_EEG:
from parallel import Parallel # Version sugerida por Fede (ver mail 02/08/2016)
q_marcas = Parallel(
) # Version sugerida por Fede (ver mail 02/08/2016)
q_marcas.setData(
0) # Solo para asegurarse de que arranque con todos los pins abajo
ponermarcas = 2
elif expInfo[EXPERIMENT_TYPE] == CONDUCTUAL:
q_marcas = 1
ponermarcas = 0
return ponermarcas, q_marcas
warnings.simplefilter("ignore")
tweets = None
sentiment_dir = "../sentiment/"
sentiment_models = {
"text_blob": find_text_blob_sentiment,
"vader": find_vader_sentiment,
}
for model_name, model_function in sentiment_models.items():
sentiment_path = os.path.join(sentiment_dir, model_name) + ".pickle"
if not os.path.exists(sentiment_path):
if tweets is None:
tweets = load_tweets()
tweets = list(tweets.items())
results = Parallel(find_text_blob_sentiment, tweets, model_name)
sentiment = {tweet_id: value for tweet_id, value in results}
save_pickle(sentiment, sentiment_path)
model_name = "flair"
sentiment_path = os.path.join(sentiment_dir, model_name) + ".pickle"
if not os.path.exists(sentiment_path):
if tweets is None:
tweets = load_tweets()
tweets = list(tweets.items())
sentiment = find_flair_sentiment(tweets, chunk_len=100000)
sentiment_models[model_name] = sentiment
save_pickle(sentiment, sentiment_path)
def process_mentions():
"""retrieves all mentions and generates captions for those who are fighting
fit"""
if not cfg('twitter:user_requests:bool'):
return
params = dict(count=200)
sources_whitelist = cfg('twitter:sources_whitelist:list')
mention_prefix = '@%s ' % twitter.me.screen_name.lower()
try:
with open('state_mentions_timeline.txt') as fp:
since_id = int(fp.read())
utils.logging.info('State: since_id=%d', since_id)
params['since_id'] = since_id
except Exception as exc:
utils.logging.warning("There's no last id saved, so I will save the "
'last id I see and then quit.')
since_id = None
filtered_statuses = []
statuses = [
status for page in tweepy.Cursor(twitter.api.mentions_timeline, **
params).pages() for status in page
]
# they are in reverse chronological order, so put them straight
statuses = statuses[::-1]
if not since_id:
since_id = statuses[-1].id
with open('state_mentions_timeline.txt', 'wt') as fp:
fp.write(str(since_id))
utils.logging.info('New since_id=%d. Goodbye!', since_id)
return
for status in statuses:
# ignore mentions that are not directed at me
if not status.text.lower().startswith(mention_prefix):
continue
# ignore retweets
if hasattr(status, 'retweeted_status'):
continue
# if the sources whitelist is enabled, ignore those who aren't on it
if (sources_whitelist and status.source not in sources_whitelist):
continue
# store this status
filtered_statuses.append(status)
if filtered_statuses:
utils.logging.info('Retrieved %d new mentions (from %d to %d).',
len(filtered_statuses), filtered_statuses[0].id,
filtered_statuses[-1].id)
with open('state_mentions_timeline.txt', 'wt') as fp:
fp.write(str(filtered_statuses[-1].id))
Akari.warmup()
parallel = Parallel(process_request, filtered_statuses,
cfg('twitter:process_threads:int') or 3)
parallel.start()
else:
utils.logging.info('Retrieved no new mentions.')
print CommandSequence([Loop('x', 1, 10, 0.5, Comment("Hello"))])
print CommandSequence(
[Loop('x', 1, 10, 0.5, Comment("Hello"), completion=True)])
print CommandSequence([
Loop('x',
1,
10,
0.5, [Comment("Hello"), Comment("World")],
completion=True,
timeout=10)
])
print CommandSequence([
Loop('x', 2, 20, 5, [
Loop('y',
1,
10,
0.5, [Comment("Hello"), Comment("World")],
completion=True,
timeout=10)
])
])
print CommandSequence([
Parallel(Loop('x', 1, 10, 0.5, Comment("Hello")),
Loop('y', 1, 10, 0.5, Comment("There")))
])
dataset = Dataset(dataset_name=args.dataset)
mlp_aux = MLP_AUX(dataset, args.negative_sampling_size,
eval(args.layers), args.epochs, args.batch_size,
args.validation_split, args.user_sampling_size,
args.core_number, args.sim_threshold)
model = mlp_aux.train_model()
hits, ndcgs = evaluate_model(model, dataset.test_data,
dataset.test_negatives, 10, 1, True)
print("Hitrate: {}".format(sum(hits) / len(hits)))
print("NDCG: {}".format(sum(ndcgs) / len(ndcgs)))
elif args.network_type == 'parallel':
dataset = Dataset(dataset_name=args.dataset)
parallel = Parallel(dataset, args.negative_sampling_size,
eval(args.layers), args.epochs, args.batch_size,
args.validation_split)
model = parallel.train_model()
hits, ndcgs = evaluate_model(model, dataset.test_data,
dataset.test_negatives, 10, 1)
print("Hitrate: {}".format(sum(hits) / len(hits)))
print("NDCG: {}".format(sum(ndcgs) / len(ndcgs)))
elif args.network_type == 'parallel-aux':
dataset = Dataset(dataset_name=args.dataset)
parallel_aux = Parallel_AUX(dataset, args.negative_sampling_size,
eval(args.layers), args.epochs,
args.batch_size, args.validation_split,
args.user_sampling_size, args.core_number,
args.sim_threshold)
model = parallel_aux.train_model()
hits, ndcgs = evaluate_model(model, dataset.test_data,
def main():
# Info de la sesion
expInfo = {
NAME: 'nombre',
BIRTHDATE: 'DD/MM/AA',
HAND: 'mano',
EXPERIMENT_TYPE: 'conductual',
OPERATOR: ''
}
# Presento cuadro para rellenar
dlg = gui.DlgFromDict(expInfo, title='Formulario')
if not (dlg.OK):
core.quit()
else:
fileName = expInfo[NAME]
if not os.path.exists('./Datos/' + fileName):
os.makedirs('./Datos/' + fileName)
dataFile = open(
'./Datos/' + fileName + '/' + str(datetime.date.today()) + '_' +
fileName + '.csv', 'a')
nombreEDF = './Datos/' + fileName + "/" + str(
datetime.date.today()) + '_' + fileName
texto = expInfo[NAME] + '; ' + str(datetime.datetime.now(
)) + '; ' + expInfo[BIRTHDATE] + '; ' + expInfo[HAND] + '; ' + expInfo[
EXPERIMENT_TYPE] + '; ' + expInfo[OPERATOR]
dataFile.write(texto)
##########################
## Parametros Pantalla ##
##########################
res = [gtk.gdk.screen_width(), gtk.gdk.screen_height()]
pantCompleta = True
#win = visual.Window(res, monitor="Mi Monitor", units="pix", color=gris, colorSpace='hex', fullscr=pantCompleta)
win = visual.Window(res,
units="pix",
color=gris,
colorSpace='hex',
fullscr=pantCompleta,
monitor="testMonitor")
win.setMouseVisible(False)
###########################################
## Inicializo parametros del experimento ##
###########################################
proporcion = 0.7
pruebas = 30
Nsess = 12
# Tiempos
#StimDur = 0.184
#ISI = 0.986
StimDur = 0.404
ISI = 0.986
ponermarcas = []
if expInfo[EXPERIMENT_TYPE] == EMOTIV:
from multiprocessing import Process, Queue
import guardar
q_marcas = Queue()
p = Process(target=guardar.save_data, args=(
nombreEDF,
q_marcas,
))
p.start()
ponermarcas = 1
elif expInfo[EXPERIMENT_TYPE] == TRADITIONAL_EEG:
from parallel import Parallel # Version sugerida por Fede (ver mail 02/08/2016)
#from psychopy import parallel
# BIOSEMI
#q_marcas=parallel.ParallelPort(address=u'/dev/parport0')
#q_marcas=parallel.PParallelDLPortIO(address=888) # Chequear que este bien la direccion del puerto paralelo
q_marcas = Parallel(
) # Version sugerida por Fede (ver mail 02/08/2016)
q_marcas.setData(
0) # Solo para asegurarse de que arranque con todos los pins abajo
ponermarcas = 2
elif expInfo[EXPERIMENT_TYPE] == CONDUCTUAL:
q_marcas = 1
ponermarcas = 0
cond = pacman
stimuli = [
pacmanImage, "./estimulo/fantasma_naranja.png",
"./estimulo/fantasma_rosado.png", "./estimulo/fantasma_verde.png",
"./estimulo/fantasma_azul.png"
]
pantalla_inicio = "./estimulo/pantini_pacman.png"
run_training(win, proporcion, 10, 6, StimDur, ISI, res, gris, negro,
blanco, stimuli, pantalla_inicio)
run_experiment(dataFile, win, proporcion, pruebas, Nsess, StimDur, ISI,
q_marcas, ponermarcas, res, gris, negro, blanco, stimuli,
pantalla_inicio, cond)
cond = angry
stimuli = [
birdImage, "./estimulo/cerdo_naranja.png",
"./estimulo/cerdo_rosado.png", "./estimulo/cerdo_verde.png",
"./estimulo/cerdo_azul.png"
]
pantalla_inicio = "./estimulo/pantini_angry.png"
run_training(win, proporcion, 10, 6, StimDur, ISI, res, gris, negro,
blanco, stimuli, pantalla_inicio)
run_experiment(dataFile, win, proporcion, pruebas, Nsess, StimDur, ISI,
q_marcas, ponermarcas, res, gris, negro, blanco, stimuli,
pantalla_inicio, cond)
def do_task(self):
if self.task != None:
p = Parallel()
print(p)
Parallel().run(self.task.logic)
def checkDatasetFiles(self,dsetName,catalog=None):
"""
Look for corrupted files in dataset.
@dsetName: dataset name
Note: not implemented
"""
writeCatalog = False
if not catalog:
catalog = self.readCatalog()
writeCatalog = True
wait = False
if not self.parallel_:
self.parallel_ = Parallel(16,self.queue_)
wait = True
print
print "Checking dataset",dsetName
info = catalog[dsetName]
files = info["files"]
print "Number of files: ", len(files)
toremove = []
for ifil,eifil in enumerate(files):
if ifil in toremove:
continue
for jfil,ejfil in enumerate(files[ifil+1:]):
if ifil+jfil in toremove:
continue
if eifil["name"] == ejfil["name"]:
toremove.append(ifil)
else:
iid = eifil["name"].rstrip(".root").rsplit("_",1)[-1]
jid = ejfil["name"].rstrip(".root").rsplit("_",1)[-1]
if iid == jid:
print "duplicated file index ", iid
print eifil["name"]
print ejfil["name"]
reply=ask_user("keep both? ")
if reply == "n":
if ask_user( "keep %s? " % ejfil["name"] ) == "n":
## files.pop(ifil+jfil)
toremove.append(ifil+jfil)
if ask_user( "keep %s? " % eifil["name"] ) == "n":
toremove.append(ifil)
## files.pop(ifil)
for ifile in sorted(toremove,reverse=True):
## print ifile
files.pop(ifile)
print "After duplicates removal: ", len(files)
info = catalog[dsetName]["files"] = files
for ifile,finfo in enumerate(files):
name = finfo["name"]
self.parallel_.run(SamplesManager.checkFile,[self,name,dsetName,ifile])
if wait:
self.parallel_.wait(printOutput=False)
self.parallel_ = None
if writeCatalog:
self.writeCatalog(catalog)
def run(self,
genome_files,
output_dir,
called_genes=False,
translation_table=None,
meta=False,
closed_ends=False):
"""Call genes with Prodigal.
Call genes with prodigal and store the results in the
specified output directory. For convenience, the
called_gene flag can be used to indicate genes have
previously been called and simply need to be copied
to the specified output directory.
Parameters
----------
genome_files : list of str
Nucleotide fasta files to call genes on.
called_genes : boolean
Flag indicating if genes are already called.
translation_table : int
Specifies desired translation table, use None to automatically
select between tables 4 and 11.
meta : boolean
Flag indicating if prodigal should call genes with the metagenomics procedure.
closed_ends : boolean
If True, do not allow genes to run off edges (throws -c flag).
output_dir : str
Directory to store called genes.
Returns
-------
d[genome_id] -> namedtuple(best_translation_table
coding_density_4
coding_density_11)
Summary statistics of called genes for each genome.
"""
self.called_genes = called_genes
self.translation_table = translation_table
self.meta = meta
self.closed_ends = closed_ends
self.output_dir = output_dir
make_sure_path_exists(self.output_dir)
progress_func = None
if self.verbose:
file_type = 'genomes'
self.progress_str = ' Finished processing %d of %d (%.2f%%) genomes.'
if meta:
file_type = 'scaffolds'
if len(genome_files):
file_type = ntpath.basename(genome_files[0])
self.progress_str = ' Finished processing %d of %d (%.2f%%) files.'
self.logger.info('Identifying genes within %s: ' % file_type)
progress_func = self._progress
parallel = Parallel(self.cpus)
summary_stats = parallel.run(
self._producer, self._consumer, genome_files, progress_func)
# An error was encountered during Prodigal processing, clean up.
if not summary_stats:
shutil.rmtree(self.output_dir)
return summary_stats
def fit(self, X, y, sample_weight=None): # 线性回归方法
"""
Fit linear model.
Parameters
----------
X : array-like or sparse matrix, shape (n_samples, n_features)
Training data
y : array_like, shape (n_samples, n_targets)
Target values. Will be cast to X's dtype if necessary
sample_weight : numpy array of shape [n_samples]
Individual weights for each sample
.. versionadded:: 0.17
parameter *sample_weight* support to LinearRegression.
Returns
-------
self : returns an instance of self.
"""
n_jobs_ = self.n_jobs
X, y = check_X_y(X,
y,
accept_sparse=['csr', 'csc', 'coo'],
y_numeric=True,
multi_output=True) # 检查数据并进行必要的格式转换
#------------------------------
# atleast_xd 支持将输入的数据
# 直接视为 x 维
# atleast_1d
# atleast_2d
# atleast_3d
if sample_weight is not None and np.atleast_1d(sample_weight).ndim > 1:
raise ValueError("Sample weights must be 1D array or scalar"
) # sampple_weight 必须为一维 array
X, y, X_offset, y_offset, X_scale = self._preprocess_data(
X,
y,
fit_intercept=self.fit_intercept,
normalize=self.normalize,
copy=self.copy_X,
sample_weight=sample_weight) # 对 X 数据进行中心以及归一化
if sample_weight is not None:
# Sample weight can be implemented via a simple rescaling.
X, y = _rescale_data(X, y, sample_weight) # 给数据添加 sample_weight
if sp.issparse(X):
if y.ndim < 2:
out = sparse_lsqr(X, y) # 哈哈哈！直接调用 scipy sparse linalg 的
# least square solution
self.coef_ = out[0]
self._residues = out[3]
else:
# sparse_lstsq cannot handle y with shape (M, K)
outs = Parallel(n_jobs=n_jobs_)(
delayed(sparse_lsqr)(X, y[:, j].ravel())
for j in range(y.shape[1]))
self.coef_ = np.vstack(out[0] for out in outs)
self._residues = np.vstack(out[3] for out in outs)
else:
self.coef_, self._residues, self.rank_, self.singular_ = \
linalg.lstsq(X, y) # Compute least-squares solution to equation Ax = b 哈哈哈！！
self.coef_ = self.coef_.T
if y.ndim == 1:
self.coef_ = np.ravel(self.coef_)
self._set_intercept(X_offset, y_offset, X_scale)
return self
def update(self,
nprocs=1,
factor=None,
bw_hz=None,
foi_hz=None,
fs_hz=None,
f_ord=None,
ftype=None,
n_freqs=None,
n_samples=None,
n_channels=None):
self.n_channels = n_channels if n_channels is not None else self.n_channels
self.n_freqs = n_freqs if n_freqs is not None else self.n_freqs
self.n_processes = min(Parallel.check_nprocs() -
1, self.n_freqs) if nprocs != 1 else 1
# Signal process properties
self.decimate_by = factor
self.n_samples = int(n_samples / self.decimate_by)
self.sample_rate = fs_hz / self.decimate_by if fs_hz is not None else self.sample_rate
self.bandwidth = bw_hz if bw_hz is not None else self.bandwidth
self.w_, self.H_ = self.create_filter(f_ord,
self.bandwidth / 2.0,
self.sample_rate / 2.0,
self.n_samples,
ftype='fir',
output='freq')
self.Hwin = self.H_[np.logical_and(self.w_ >= -self.bandwidth / 2.0,
self.w_ < self.bandwidth / 2.0)]
self.n_samples_procs = self.Hwin.size
# Setup center frequencies
if len(foi_hz) > 1:
cf = np.arange(*foi_hz, np.diff(foi_hz) / self.n_freqs, dtype=int)
diff = cf.shape[0] - self.n_freqs
if diff > 0:
cf = cf[:-diff]
else:
cf = foi_hz
self.freqs = np.asarray([
(f - self.bandwidth / 2, f + self.bandwidth / 2) for f in cf
])
# Create rules for how to handle the data
self._encoder_rule()
self._decoder_rule()
if self.n_processes > 1:
self.pfunc = Parallel(self.multiply,
nprocs=self.n_processes,
axis=0,
flag=0,
ins_shape=[(self.n_channels, self.n_freqs,
self.n_samples_procs),
(1, self.n_samples_procs)],
ins_dtype=[np.complex64, np.complex64],
out_shape=(self.n_channels, self.n_freqs,
self.n_samples_procs),
out_dtype=np.complex64)
# !setup! import neworder from parallel import Parallel # import our model definition #neworder.verbose() #neworder.checked(False) # must be MPI enabled assert neworder.mpi.size( ) > 1, "This configuration requires MPI with >1 process" # !setup! # !run! population_size = 100 p = 0.01 timeline = neworder.LinearTimeline(0, 10, 10) model = Parallel(timeline, p, population_size) neworder.run(model) #!run!
