def ftp_download(file_path, is_override, output_directory, uri, user, password, worker_semaphore, inserted_task_event, tasks, log_format, log_level):
"""
download a specify file from the given ftp server and output the specify directory
#>>> ftp_download('/tmp/archives.zip.002', True, 'f:/', '16.60.160.90', 'edwin', 'edwin')
#>>> os.path.isfile('f:/archives.zip.002')
#True
"""
try:
path, filename = os.path.split(file_path)
logger = multiprocessing.get_logger()
init_logger(logger, log_level, log_format)
ftp = ftplib.FTP(host=uri, user=user, passwd=password)
ftp.cwd(path)
output_file = os.path.join(output_directory, filename)
logger.info('Start downloading %s' % file_path)
if os.path.isfile(output_file) and not is_override:
return
if os.path.isfile(output_file):
os.remove(output_file)
ftp.retrbinary('RETR %s' % filename, lambda data: open(output_file, 'ab').write(data))
ftp.close()
logger.info('Complete file to %s' % output_file)
except Exception as e:
logger = multiprocessing.get_logger()
logger.error('Download %s failed, error info %s' % (file_path, e))
output_file = os.path.join(output_directory, filename)
if os.path.exists(output_file):
os.remove(output_file)
tasks.put(filename)
inserted_task_event.set()
finally:
logger = multiprocessing.get_logger()
logger.debug('Release lock %s' % id(worker_semaphore))
worker_semaphore.release()
def setup_jenkins_console_logger(level=logging.INFO):
formatter = logging.Formatter('%(process)s: %(msg)s')
# Time is logged by Jenkins. Any log files on disk will be removed by
# Jenkins when the next job starts
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
multiprocessing.get_logger().setLevel(level)
multiprocessing.get_logger().addHandler(stream_handler)
def main(args):
try:
if args[0] == "filter":
return run_filter(*args[1:])
elif args[0] == "algo":
return run_algo(*args[1:])
except:
get_logger().error(traceback.format_exc())
return False
def __init__(self):
proc_count = cpu_count() / 5 * 4
if proc_count < 1:
proc_count = 1
self.pool = multiprocessing.Pool(proc_count, initializer=setup_jenkins_console_logger)
self.workspace_path = ""
self.runlist = []
self.params = {}
self.process_cli()
multiprocessing.get_logger().info("\n{0}\nIntegration tests runner started.\n{0}\n".format("*" * 80))
def spawn_test_process(test, flags):
spell = ["{0} {1}".format(test, flags)]
process = subprocess.Popen(spell, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
multiprocessing.get_logger().info(spell[0])
out, err = process.communicate()
# We need the out for getting the list of tests from an exec file
# by sending it the --list_tests flag
return test, filter(None, out.splitlines()), err, process.returncode
def spawn_test_process(test, flags):
spell = ["{0} {1}".format(test, flags)]
process = subprocess.Popen(spell,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True
)
multiprocessing.get_logger().info(spell[0])
_, err = process.communicate()
return test, err, process.returncode
def __call__(self, *args, **kwargs):
try:
result = self.__callable(*args, **kwargs)
except Exception as e:
# Here we add some debugging help. If multiprocessing's
# debugging is on, it will arrange to log the traceback
multiprocessing.get_logger().error(traceback.format_exc())
# Re-raise the original exception so the Pool worker can
# clean up
raise
# It was fine, give a normal answer
return result
def loop(self):
while True:
multiprocessing.get_logger().info("%s server: Waiting for signal" % self.sessionID)
self.event.wait()
while not self.queue.empty():
print("%s server: Got request signal", self.sessionID)
self.event.clear()
request = queues[x].get(False)
if isinstance(request, basestring) and request.lower() == "terminate":
# terminate signal received => save dataframe and exit event loop => process gets terminated
self.dataFrameManager.saveDataFrame()
break
else:
self.requestManager.processRequest(request)
def run_cross_tests(server_match, client_match, jobs, skip_known_failures):
logger = multiprocessing.get_logger()
logger.debug('Collecting tests')
with open(path_join(TEST_DIR, CONFIG_FILE), 'r') as fp:
j = json.load(fp)
tests = crossrunner.collect_cross_tests(j, server_match, client_match)
if not tests:
print('No test found that matches the criteria', file=sys.stderr)
print(' servers: %s' % server_match, file=sys.stderr)
print(' clients: %s' % client_match, file=sys.stderr)
return False
if skip_known_failures:
logger.debug('Skipping known failures')
known = crossrunner.load_known_failures(TEST_DIR)
tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
dispatcher = crossrunner.TestDispatcher(TEST_DIR, TEST_DIR, jobs)
logger.debug('Executing %d tests' % len(tests))
try:
for r in [dispatcher.dispatch(test) for test in tests]:
r.wait()
logger.debug('Waiting for completion')
return dispatcher.wait()
except (KeyboardInterrupt, SystemExit):
logger.debug('Interrupted, shutting down')
dispatcher.terminate()
return False
def download_wallpapers(r, task, image_dir, resolution, quantity):
"""Downloads a number of wallpapers based on the resolution and quantity
and adds each new wallpaper to the Redis database. For any wallpapers that
are 1080p a low resolution version is created and the pair of images
is added to the list of background images.
"""
logger = get_logger()
wall = WallDownloader(image_dir)
for wallpaper in wall.downloads(resolution, quantity):
logger.info("Wallpaper: " + wallpaper + " downloaded!")
r.incrbyfloat('job:' + task + ':progress', 75.0 / quantity)
image_name = WallDownloader.get_filename(wallpaper)
uuid = r.get('image:' + image_name + ':uuid')
if not uuid:
uuid = generate_uuid()
r.set('image:' + image_name + ':uuid', uuid)
# Add the image to the list for the current resolution
r.set('image:' + uuid + ':' + resolution, wallpaper)
r.sadd('image:' + resolution + ':uuids', uuid)
# Add 1080p images to list of background images
if resolution == '1920x1080':
create_background(r, task, uuid, image_dir, image_name)
def prepare_download(task, image_dir, output_dir):
"""Prepares a download request submitted by a user, fetches a random set
of wallpapers based on the resolution and number of wallpapers specified and
creates a compressed zip file for download.
"""
# Connect to the local Redis database, get logger
r = redis.StrictRedis(host='localhost')
logger = get_logger()
# Get the resolution and number of downloads requested
resolution = r.get('job:' + task + ':resolution')
quantity = int(r.get('job:' + task + ':quantity'))
# Exit if the job timed out (resolution and quantity requested no longer exist)
# This should not happen unless the system is under heavy load, log issue
if not resolution or not quantity:
logger.warning('Job: ' + task + ' No resolution or quantity, job expired?')
sys.exit(1)
# Download more wallpapers if there are not enough for the resolution
if r.scard('image:' + resolution + ':uuids') < max_wallpapers:
logger.info('Job: ' + task + ' Downloading: ' + resolution + ' wallpapers!')
download_wallpapers(r, task, image_dir, resolution, quantity)
# Get a random set of wallpapers and create a zip file
wallpapers = []
for uuid in r.srandmember('image:' + resolution + ':uuids', quantity):
wallpapers.append(r.get('image:' + uuid + ':' + resolution))
logger.info('Job: ' + task + ' Wallpapers: ' + ', '.join(wallpapers))
# Create a zip file, set key in redis
compress_wallpapers(r, task, output_dir, wallpapers)
sys.exit(0)
def create_background(r, task, uuid, image_dir, image_name):
"""Creates a low resolution version of a 1080p wallpaper provided and saves
the matching pair of images to the list of background images that are
randomly displayed on the website.
"""
logger = get_logger()
# Get the wallpaper file path
uuid = r.get('image:' + image_name + ':uuid')
file = r.get('image:' + uuid + ':1920x1080')
new_img_dir = os.path.join(os.path.normpath(image_dir), '800x480')
new_file = os.path.join(os.path.normpath(new_img_dir), image_name + '_800x480.jpg')
if not os.path.exists(new_img_dir):
os.makedirs(new_img_dir)
# Resize the image as 800x480 at 50% quality, save as new image
logger.info('Job: ' + task + ' UUID: ' + uuid + ', creating background: ' + new_file)
with Image(filename=file) as img:
with img.clone() as new_img:
new_img.compression_quality = 50
new_img.resize(800, 480)
new_img.save(filename=new_file)
# Add the new background to the list
r.set('image:' + uuid + ':800x480', new_file)
r.sadd('image:backgrounds:uuids', uuid)
def run_periodic_tasks(self):
logger = get_logger()
applied = default_periodic_status_backend.run_periodic_tasks()
for task, task_id in applied:
logger.debug(
"PeriodicWorkController: Periodic task %s applied (%s)" % (
task.name, task_id))
def wrapped_function(*args, **kwargs):
global return_value
logger = multiprocessing.get_logger()
# create a pipe to retrieve the return value
parent_conn, child_conn = multiprocessing.Pipe()
# create and start the process
subproc = multiprocessing.Process(target=subprocess_func, name=" multiproc function call", args=(func,
child_conn,
mem_in_mb,
cpu_time_in_s,
wall_time_in_s,
num_processes) + args,
kwargs=kwargs)
logger.debug("Your function is called now.")
return_value = None
# start the process
subproc.start()
child_conn.close()
try:
# read the return value
return_value = parent_conn.recv()
except EOFError: # Don't see that in the unit tests :(
logger.debug("Your function call closed the pipe prematurely -> None will be returned")
return_value = None
except:
raise
finally:
# don't leave zombies behind
subproc.join()
return (return_value);
def invoke_cmd_worker(item):
try:
logger = multiprocessing.get_logger()
pid = multiprocessing.current_process().pid
plugin_dir, plugin, filepath, events_limit = item
worker_fpath = os.path.abspath(__file__)
cmd = 'gzip -d -c %s | python2.7 %s %s %s %s' % (
filepath, worker_fpath, plugin_dir, plugin, events_limit
)
logger.info(
'%d: Starting job: %s', pid, cmd
)
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, shell=True,
env=env
)
output = process.communicate()[0]
return output
except Exception as e:
traceback.print_exc(e)
def run_feature_tests(server_match, feature_match, jobs, skip_known_failures, only_known_failures, retry_count, regex):
basedir = path_join(ROOT_DIR, FEATURE_DIR_RELATIVE)
logger = multiprocessing.get_logger()
logger.debug('Collecting tests')
with open(path_join(TEST_DIR, CONFIG_FILE), 'r') as fp:
j = json.load(fp)
with open(path_join(basedir, CONFIG_FILE), 'r') as fp:
j2 = json.load(fp)
tests = crossrunner.collect_feature_tests(j, j2, server_match, feature_match, regex)
if not tests:
print('No test found that matches the criteria', file=sys.stderr)
print(' servers: %s' % server_match, file=sys.stderr)
print(' features: %s' % feature_match, file=sys.stderr)
return False
if only_known_failures:
logger.debug('Only running known failures')
known = crossrunner.load_known_failures(basedir)
tests = list(filter(lambda t: crossrunner.test_name(**t) in known, tests))
if skip_known_failures:
logger.debug('Skipping known failures')
known = crossrunner.load_known_failures(basedir)
tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
dispatcher = crossrunner.TestDispatcher(TEST_DIR, ROOT_DIR, FEATURE_DIR_RELATIVE, jobs)
logger.debug('Executing %d tests' % len(tests))
try:
for r in [dispatcher.dispatch(test, retry_count) for test in tests]:
r.wait()
logger.debug('Waiting for completion')
return dispatcher.wait()
except (KeyboardInterrupt, SystemExit):
logger.debug('Interrupted, shutting down')
dispatcher.terminate()
return False
def __init__(self,cmd_param):
'''
主进程类 为了兼容windowsè€Œè¿™æ ·åˆ›å»ºç±»
'''
super(Mtask,self).__init__()
self.start_time = datetime.datetime.now()
self.parent_timeout = 180
self.parent_timeout_flag = 0
self.child_timeout =120
self.child_num = 10
self.slice_num = 20
self.process_list = []
self.result =[]
self.batch_id = 0
self.print_flag = 1
self.mult_debug_flag = 0
self.cmd_param = cmd_param
if self.mult_debug_flag:
#设置进程log日志
multiprocessing.log_to_stderr()
logger=multiprocessing.get_logger()
logger.setLevel(logging.INFO)
def SumData(roi_start_list, roi_end_list, file_list, file_path, istart, iend, queue):
logger = mp.get_logger()
mp.log_to_stderr(logging.INFO)
roi_sums = mproc.SHARED_ARRAY
data_array = XYDataArray()
num_rois = len(roi_start_list)
logger.info("Reading files from %d to %d" % (istart, iend))
if istart > 0:
istart = istart - 1
# Process each file in the list that falls in the range istart to iend
for i in range(istart, iend):
# Read in the information from the file and create numpy arrays from that information.
data_array.CreateArrays(os.path.join(file_path, file_list[i]))
# Sum the data in the arrays that lies between the roi values. Do this
# for each roi that was created.
for j in range(num_rois):
# logger.info("Summing roi %d from file %d" % (j, i))
roi_sums[j][i] = roi_sums[j][i] + data_array.SumROIData(roi_start_list[j], roi_end_list[j])
# Add a value of 1 to the queue so that the user interface can be updated
# with the latest progress.
queue.put(1)
return roi_sums
def initialize(online, version, use_cherrypy):
# configure the logger
global logger
if use_cherrypy:
import cherrypy
logger = cherrypy.log
else:
import multiprocessing
logger = multiprocessing.get_logger()
# create a database session
connectionString = ConnectionString()
engine = None
if online == 'True':
engine = create_engine(connectionString.connectUrlonline)
else:
engine = create_engine(connectionString.connectUrloffline)
Base.prepare(engine, reflect=True)
session = scoped_session(sessionmaker(engine))
# store the version id
global databuilder
databuilder = DataBuilder(session, version, logger)
def run_cross_tests(server_match, client_match, jobs, skip_known_failures, retry_count, regex):
logger = multiprocessing.get_logger()
logger.debug("Collecting tests")
with open(path_join(TEST_DIR, CONFIG_FILE), "r") as fp:
j = json.load(fp)
tests = crossrunner.collect_cross_tests(j, server_match, client_match, regex)
if not tests:
print("No test found that matches the criteria", file=sys.stderr)
print(" servers: %s" % server_match, file=sys.stderr)
print(" clients: %s" % client_match, file=sys.stderr)
return False
if skip_known_failures:
logger.debug("Skipping known failures")
known = crossrunner.load_known_failures(TEST_DIR)
tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
dispatcher = crossrunner.TestDispatcher(TEST_DIR, ROOT_DIR, TEST_DIR_RELATIVE, jobs)
logger.debug("Executing %d tests" % len(tests))
try:
for r in [dispatcher.dispatch(test, retry_count) for test in tests]:
r.wait()
logger.debug("Waiting for completion")
return dispatcher.wait()
except (KeyboardInterrupt, SystemExit):
logger.debug("Interrupted, shutting down")
dispatcher.terminate()
return False
def __init__(self, host, pipe, port=80, channels=None):
"""
Create a new client.
host : host to connect
pipe : pipe of paths
port : port to connect
channels : map of file descriptors
"""
asynchat.async_chat.__init__(self, map=channels)
self._log = multiprocessing.get_logger()
self._host = host
self._pipe = pipe
self._port = port
self._time = 0
self._htime = 0
self._path = ""
self._header = ""
self._body = ""
self._data = ""
self._protocol = ""
self._status = -1
self._status_msg = ""
self._close = False
self._chunked = True
self._content_length = -1
self.set_terminator(HTTPAsyncClient.TERMINATOR)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((self._host, self._port))
self._log.debug(self.logmsg("HTTPAsyncClient connected to %s:%d",
self._host, self._port))
self.send_request()
def test_setup_logger_no_handlers_file(self):
from multiprocessing import get_logger
l = get_logger()
l.handlers = []
tempfile = mktemp(suffix="unittest", prefix="celery")
l = setup_logger(logfile=tempfile, loglevel=0)
self.assertTrue(isinstance(l.handlers[0], logging.FileHandler))
def error(msg, *args): """Shortcut to multiprocessing's logger""" ############################################################# DEBUG import sys sys.stdout.flush() ############################################################# DEBUG return mp.get_logger().error(msg, *args)
def setup_logger(settings):
class ColorFilter(logging.Filter):
def filter(self, record):
if not hasattr(record, 'xcolor'):
record.xcolor = ''
return True
logger = multiprocessing.get_logger()
logger.setLevel(logging.DEBUG)
if settings['log_file_dir'] and settings['log_file_level']:
pname = multiprocessing.current_process().name
ctime = time.strftime('%Y-%m-%d_%H-%M-%S')
filename = os.path.join(settings['log_file_dir'],
'log_%s_%s.txt' % (pname, ctime))
filehandler = logging.FileHandler(filename, mode='w', encoding='utf8')
filehandler.setLevel(settings['log_file_level'])
fileformatter = logging.Formatter(
'[%(asctime)s] %(levelname)-8s %(message)s')
filehandler.setFormatter(fileformatter)
logger.addHandler(filehandler)
if settings['log_stderr_level']:
streamhandler = logging.StreamHandler(stream=sys.stderr)
streamhandler.setLevel(settings['log_stderr_level'])
streamhandler.addFilter(ColorFilter())
streamformatter = logging.Formatter(
'[%(levelname)s/%(processName)s] %(xcolor)s%(message)s' +
Colors.NORMAL)
streamhandler.setFormatter(streamformatter)
logger.addHandler(streamhandler)
return logger
def test_log():
multiprocessing.log_to_stderr()
logger = multiprocessing.get_logger()
logger.setLevel(logging.INFO)
t1 = time.time()
print(time.time() - t1)
logger.info("done")
def smac_classpath():
"""
Small function gathering all information to build the java class path.
:returns: string representing the Java classpath for SMAC
"""
import multiprocessing
from pkg_resources import resource_filename
logger = multiprocessing.get_logger()
smac_folder = resource_filename("pysmac", 'smac/%s' % pysmac.remote_smac.SMAC_VERSION)
smac_conf_folder = os.path.join(smac_folder, "conf")
smac_patches_folder = os.path.join(smac_folder, "patches")
smac_lib_folder = os.path.join(smac_folder, "lib")
classpath = [fname for fname in os.listdir(smac_lib_folder) if fname.endswith(".jar")]
classpath = [os.path.join(smac_lib_folder, fname) for fname in classpath]
classpath = [os.path.abspath(fname) for fname in classpath]
classpath.append(os.path.abspath(smac_conf_folder))
classpath.append(os.path.abspath(smac_patches_folder))
# For Windows compability
classpath = (os.pathsep).join(classpath)
logger.debug("SMAC classpath: %s", classpath)
return classpath
def run(self):
logger = multiprocessing.get_logger()
GPIO.setmode(GPIO.BOARD)
GPIO.setup(self.led_pin, GPIO.OUT)
GPIO.output(self.led_pin, True)
reader = MFRC522(self.spi_device)
while True:
reader.MFRC522_Request(reader.PICC_REQIDL)
(status, data) = reader.MFRC522_Anticoll()
if status == reader.MI_OK:
vote = {
'tag_id': "%0.2x%0.2x%0.2x%0.2x" % (data[0], data[1], data[2], data[3]),
'timestamp': time.time(),
'voting_machine_id': getnode(), # see RFC 4122
'spi_device': self.spi_device,
}
if self.submit_vote(vote):
logger.info('Vote submitted %s', vote)
self.led_blink()
else:
logger.error('Could not submit vote %s', vote)
def run(plugin_name, start_date, end_date, plugin_dir,
data_dir='/mnt/disk1/alohalytics/by_date',
results_dir='./stats',
events_limit=0):
"""
Pyaloha stats processing pipeline:
0. Load worker, aggregator, processor classes from a specified plugin (script)
1. Run workers (data preprocessors) on alohalytics files within specified range
2. Accumulate [and postprocess] worker results with an aggregator instance
3. Run stats processor and print results to stdout
"""
aggregator = aggregate_raw_data(
data_dir, results_dir, plugin_dir, plugin_name,
start_date, end_date, events_limit
)
stats = load_plugin(
plugin_name, plugin_dir=plugin_dir
).StatsProcessor(aggregator)
logger = multiprocessing.get_logger()
logger.info('Stats: processing')
stats.process_stats()
logger.info('Stats: outputting')
stats.print_stats()
logger.info('Stats: done')
def SumPixels(file_list, file_path, istart, iend, queue):
logger = mp.get_logger()
mp.log_to_stderr(logging.INFO)
roi_sums = mproc.SHARED_ARRAY
data_array = TiffDatatArray()
logger.info("Reading files from %d to %d" % (istart, iend))
if istart > 0:
istart = istart - 1
# Process each file in the list that falls in the range istart to iend
for i in range(istart, iend):
# Read in the information from the file and create numpy arrays from that information.
data_array.CreateArrays(os.path.join(file_path, file_list[i]))
# Sum the data in the arrays that lies between the roi values. Do this
# for each roi that was created.
new_sum = data_array.GetDataArray()
roi_sums = numpy.add(roi_sums, new_sum)
# Add a value of 1 to the queue so that the user interface can be updated
# with the latest progress.
queue.put(1)
return roi_sums
def setup_logger(logFormat, logFileName):
logger = multiprocessing.get_logger()
logHandler = logging.FileHandler(logFileName, mode = 'a')
logHandler.setFormatter(logging.Formatter(logFormat))
logger.addHandler(logHandler)
logger.setLevel(logging.DEBUG)
return logger
import multiprocessing as mp
import ctypes
import argparse
import numpy as np
import scipy.linalg
import time
import logging
info = mp.get_logger().info
#================================================================== Standard data as arg
def Estep_serial(X, w, MuList, SigmaList):
N = X.shape[0]
K = w.shape[0]
logResp = np.zeros((N, K))
for k in xrange(K):
logResp[:, k] = loggausspdf(X, MuList[k, :], SigmaList[k, :, :])
logResp += np.log(w)
return logResp
def distMahal(X, mu, Sigma):
''' Calc mahalanobis distance: (x-mu)^T Sigma^{-1} (x-mu)
for each row of matrix X
'''
Xdiff = X - mu
cholSigma = scipy.linalg.cholesky(Sigma, lower=True)
Q = np.linalg.solve(cholSigma, Xdiff.T)
distPerRow = np.sum(Q**2, axis=0)
import logging
from typing import List
import sys
import sqlite3
import multiprocessing
from multiprocessing import Process
from allennlp.common.file_utils import cached_path
logger = logging.getLogger(__name__) # pylint: disable=invalid-name
MULTIPROCESSING_LOGGER = multiprocessing.get_logger()
class SqlExecutor:
"""
This class evaluates SQL queries by connecting to a SQLite database. Because SQLite is disk-based
we just need to provide one file with the location. We execute the predicted SQL query and the labeled
queries against the database and check if they execute to the same table.
"""
def __init__(self, database_file: str) -> None:
# Initialize a cursor to our sqlite database, so we can execute SQL queries for denotation accuracy.
self._database_file = cached_path(database_file)
self._connection = sqlite3.connect(self._database_file)
self._cursor = self._connection.cursor()
def evaluate_sql_query(self, predicted_sql_query: str,
sql_query_labels: List[str]) -> int:
# We set the logging level for the subprocesses to warning, otherwise, it will
# log every time a process starts and stops.
MULTIPROCESSING_LOGGER.setLevel(logging.WARNING)
def wrapper(*largs, **kwargs):
try:
res = f(*largs, **kwargs)
except Exception, e:
multiprocessing.get_logger().exception(e)
raise
def __init__(self):
self._log = multiprocessing.get_logger()
self._lock = multiprocessing.Lock()
self._ports = set()
self._dom_ports = set()
self._last_alloc = 0
def main():
# send it all to stderr
mp.log_to_stderr()
# get access to a logger and set its logging level to INFO
logger = mp.get_logger()
logger.setLevel(logging.INFO)
dataset = read_data(DATASET_PATH)
# global train_x, test_x, train_y, test_y
train_x, test_x, train_y, test_y = split_dataset(dataset, 0.25)
# print("--- Testing Sequence DE ---")
# start_time_seq = time.time()
# result_seq = list(de_sequence(fobj, bounds=[(-100, 100)] * 6))
# print(result_seq[-1])
# print("")
# print("--- %s seconds ---" % (time.time() - start_time_seq))
#
# sleep(5)
print("--- Tuning Random Forest with Parallel DE ---")
start_time_rf_tuning_para = time_RF.time()
# result_para = list(de_parallel(fobj, bounds=[(-100, 100)] * 6))
# print(result_para[-1])
# initialization
bounds = [(10, 150), (1, 20), (2, 20), (2, 50), (0.01, 1), (1, 10)]
mut = 0.8
crossp = 0.7
popsize = 60
its = 100
dimensions = len(bounds)
pop = np.random.rand(popsize, dimensions)
# pdb.set_trace()
min_b, max_b = np.asarray(bounds).T
diff = np.fabs(min_b - max_b)
pop_denorm = min_b + pop * diff
# convert from float to integer
pop_denorm_convert = pop_denorm.tolist()
result_list = []
temp_list = []
for index in pop_denorm_convert:
temp_list.append(np.int_(np.round_(index[0])))
temp_list.append(np.int_(np.round_(index[1])))
temp_list.append(np.int_(np.round_(index[2])))
temp_list.append(np.int_(np.round_(index[3])))
temp_list.append(float('%.2f' % index[4]))
temp_list.append(np.int(np.round_(index[5])))
result_list.append(temp_list)
temp_list = []
fitness = np.asarray([
rf_tuning(index[0], index[1], index[2], index[3], index[4], index[5],
train_x, test_x, train_y, test_y) for index in result_list
])
best_idx = np.argmax(fitness)
best = pop_denorm[best_idx]
print("Dimension:", dimensions)
print("pop:", pop)
print("min_b:", min_b)
print("max_b:", max_b)
print("diff:", diff)
print("pop_denorm:", pop_denorm)
print("fitness:", fitness)
print("best_idx:", best_idx)
print("best:", best)
lock = mp.Lock()
# execute loops in each process
processes = []
for x in range(mp.cpu_count()):
processes.append(
mp.Process(target=de_innerloop,
args=(output, its, popsize, pop, mut, dimensions,
crossp, min_b, diff, lock, fitness, best_idx,
best, train_x, test_x, train_y, test_y)))
# Run processes
for p in processes:
p.start()
# Exit the completed processes
# Without join() function call, process will remain idle and won’t terminate
for p in processes:
p.join()
# Get process results from the output queue
results = [output.get() for p in processes]
print(results)
print("")
print("--- %s seconds ---" % (time_RF.time() - start_time_rf_tuning_para))
print("")
from bs4 import BeautifulSoup as BS
from urllib2 import urlopen
from urlparse import urlparse
from multiprocessing import Pool, get_logger
from functools import partial
import os.path as op
import os
import logging.handlers
import sys
console_handler = logging.StreamHandler(sys.stdout)
console_formatter = logging.Formatter(
"[%(asctime)s] %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S")
console_handler.setFormatter(console_formatter)
get_logger().addHandler(console_handler)
get_logger().setLevel(logging.INFO)
TUMBLR_NAME = sys.argv[1]
BASE_URL = 'http://{}.tumblr.com'.format(TUMBLR_NAME)
def download_photoset(i, photoset_paths, page):
try:
len_posts = len(photoset_paths)
post_path = photoset_paths[i]
u = urlopen(BASE_URL + post_path)
bs = BS(u, 'html.parser')
post_imgs = [
a.attrs['href'] for a in bs.find_all('a') if a.attrs['href']
]
def __init_log(self):
# multiprocessing.log_to_stderr(logging.DEBUG)
multiprocessing.log_to_stderr()
self.logger = multiprocessing.get_logger()
self.logger.setLevel(logging.CRITICAL)
import time
import math
import multiprocessing
import logging
import pandas as pd
from multiprocessing import Process, current_process
from multiprocessing import log_to_stderr, get_logger
import argparse
if __name__ == '__main__':
parser = argparse.ArgumentParser()
#logging info
log_to_stderr()
logger = get_logger()
logger.setLevel(logging.INFO)
#optional arguments
parser.add_argument('--pout',
help='Output file directory for candidate h5',
type=str,
default='/home/guest/suyash/default_cand/')
parser.add_argument('--pin',
help='input csv file for candmaker',
type=str,
default='/home/guest/suyash/fetch/cands.csv')
parser.add_argument('--clean',
help='deletes contents of default',
default=0)
args = parser.parse_args()
def main():
parser = argparse.ArgumentParser(prog="sir")
parser.add_argument("-d", "--debug", action="store_true")
parser.add_argument("--sqltimings", action="store_true")
subparsers = parser.add_subparsers()
reindex_parser = subparsers.add_parser("reindex",
help="Reindexes all or a single "
"entity type")
reindex_parser.set_defaults(func=reindex)
reindex_parser.add_argument('--entity-type',
action='append',
help="Which entity types to index.",
choices=SCHEMA.keys())
generate_trigger_parser = subparsers.add_parser("triggers",
help="Generate triggers")
generate_trigger_parser.set_defaults(func=generate_func)
generate_trigger_parser.add_argument('-t',
'--trigger-file',
action="store",
default="sql/CreateTriggers.sql",
help="The filename to save the "
"triggers into")
generate_trigger_parser.add_argument('-f',
'--function-file',
action="store",
default="sql/CreateFunctions.sql",
help="The filename to save the "
"functions into")
generate_trigger_parser.add_argument('-bid',
'--broker-id',
action="store",
default="1",
help="ID of the AMQP broker row "
"in the database.")
generate_extension_parser = subparsers.add_parser(
"extension", help="Generate extension")
generate_extension_parser.set_defaults(func=generate_extension)
generate_extension_parser.add_argument('-e',
'--extension-file',
action="store",
default="sql/CreateExtension.sql",
help="The filename to save the "
"extension into")
amqp_setup_parser = subparsers.add_parser("amqp_setup",
help="Set up AMQP exchanges and "
"queues")
amqp_setup_parser.set_defaults(func=setup_rabbitmq)
amqp_watch_parser = subparsers.add_parser("amqp_watch",
help="Watch AMQP queues for "
"changes")
amqp_watch_parser.set_defaults(func=watch)
args = parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
loghandler = logging.StreamHandler()
if args.debug:
formatter = logging.Formatter(fmt="%(processName)s %(asctime)s "
"%(levelname)s: %(message)s")
else:
formatter = logging.Formatter(fmt="%(asctime)s: %(message)s")
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
mplogger = multiprocessing.get_logger()
mplogger.setLevel(logging.ERROR)
mplogger.addHandler(loghandler)
if args.sqltimings:
from sqlalchemy import event
from sqlalchemy.engine import Engine
import time
sqltimelogger = logging.getLogger("sqltimer")
sqltimelogger.setLevel(logging.DEBUG)
sqltimelogger.addHandler(loghandler)
@event.listens_for(Engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement, parameters, context,
executemany):
conn.info.setdefault('query_start_time', []).append(time.time())
sqltimelogger.debug("Start Query: %s", statement)
sqltimelogger.debug("With Parameters: %s", parameters)
@event.listens_for(Engine, "after_cursor_execute")
def after_cursor_execute(conn, cursor, statement, parameters, context,
executemany):
total = time.time() - conn.info['query_start_time'].pop(-1)
sqltimelogger.debug("Query Complete!")
sqltimelogger.debug("Total Time: %f", total)
config.read_config()
try:
init_raven_client(config.CFG.get("sentry", "dsn"))
except ConfigParser.Error as e:
logger.info(
"Skipping Raven client initialization. Configuration issue: %s", e)
func = args.func
args = vars(args)
func(args)
def init(loglevel=logging.INFO):
multiprocessing.log_to_stderr()
multiprocessing.get_logger().setLevel(loglevel)
results = multiprocessing.Queue()
return results
def synchronous_pull_with_lease_management(project_id, subscription_name):
"""Pulling messages synchronously with lease management"""
# [START pubsub_subscriber_sync_pull_with_lease]
import logging
import multiprocessing
import random
import time
from google.cloud import pubsub_v1
# TODO project_id = "Your Google Cloud Project ID"
# TODO subscription_name = "Your Pub/Sub subscription name"
subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(project_id,
subscription_name)
NUM_MESSAGES = 2
ACK_DEADLINE = 30
SLEEP_TIME = 10
# The subscriber pulls a specific number of messages.
response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES)
multiprocessing.log_to_stderr()
logger = multiprocessing.get_logger()
logger.setLevel(logging.INFO)
def worker(msg):
"""Simulates a long-running process."""
RUN_TIME = random.randint(1, 60)
logger.info("{}: Running {} for {}s".format(
time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME))
time.sleep(RUN_TIME)
# `processes` stores process as key and ack id and message as values.
processes = dict()
for message in response.received_messages:
process = multiprocessing.Process(target=worker, args=(message, ))
processes[process] = (message.ack_id, message.message.data)
process.start()
while processes:
for process in list(processes):
ack_id, msg_data = processes[process]
# If the process is still running, reset the ack deadline as
# specified by ACK_DEADLINE once every while as specified
# by SLEEP_TIME.
if process.is_alive():
# `ack_deadline_seconds` must be between 10 to 600.
subscriber.modify_ack_deadline(
subscription_path,
[ack_id],
ack_deadline_seconds=ACK_DEADLINE,
)
logger.info("{}: Reset ack deadline for {} for {}s".format(
time.strftime("%X", time.gmtime()),
msg_data,
ACK_DEADLINE,
))
# If the processs is finished, acknowledges using `ack_id`.
else:
subscriber.acknowledge(subscription_path, [ack_id])
logger.info("{}: Acknowledged {}".format(
time.strftime("%X", time.gmtime()), msg_data))
processes.pop(process)
# If there are still processes running, sleeps the thread.
if processes:
time.sleep(SLEEP_TIME)
print("Received and acknowledged {} messages. Done.".format(
len(response.received_messages)))
subscriber.close()
class MegaMerger(multiprocessing.Process):
log = multiprocessing.get_logger()
def __init__(self, input_file_queue, output_file, ninputs):
super(MegaMerger, self).__init__()
self.input = input_file_queue
self.output = output_file
self.first_merge = True
self.ninputs = ninputs
self.processed = 0
self.pbar = ProgressBar(widgets=[
FormatLabel('Processed %(value)i/' + str(ninputs) + ' files. '),
ETA(),
Bar('>')
],
maxval=ninputs).start()
self.pbar.update(0)
self.files_to_clean = None
def merge_into_output(self, files):
self.files_to_clean = list(files)
self.log.info("Merging %i into output %s", len(files), self.output)
# Merge into a temporary output file
output_file_hash = hashlib.md5()
to_merge = []
for file in files:
to_merge.append(file)
output_file_hash.update(file)
output_file_name = os.path.join(tempfile.gettempdir(),
output_file_hash.hexdigest() + '.root')
# If we are doing a later merge, we need to include the
# "merged-so-far"
if self.first_merge:
self.first_merge = False
else:
to_merge.append(self.output)
merger = ROOT.TFileMerger()
merger.OutputFile(output_file_name)
for file in to_merge:
merger.AddFile(file, False)
result = merger.Merge()
self.log.info("Merge completed with result: %s" % result)
self.log.info("Output file is: %s, moving to %s", output_file_name,
self.output)
shutil.move(output_file_name, self.output)
# Cleanup. We don't need to cleanup the temporary output, since it
# is moved.
while self.files_to_clean:
os.remove(self.files_to_clean.pop())
return result
def run(self):
# ignore sigterm signal and let parent take care of this
signal.signal(signal.SIGINT, signal.SIG_IGN)
while True:
# Check if we are done merging
done = False
inputs_to_merge = []
# Accumulate some files to merge
while True:
try:
self.log.debug("trying to get")
to_merge = self.input.get(timeout=1)
self.log.debug("got %s", to_merge)
# Check for poison pill
if to_merge is None:
self.log.info("Got poison pill - shutting down")
done = True
break
inputs_to_merge.append(to_merge)
self.processed += to_merge[0]
self.pbar.update(self.processed)
# Make it merge the files if the queue is stacking up.
if len(inputs_to_merge) > 15:
break
except Empty:
self.log.debug("empty to get")
# Noting to merge right now
break
except IOError, e:
if e.errno == errno.EINTR:
self.log.debug(
"Interrupted by IOError, probably because user's system setting changed in a weird way"
)
break
else:
raise
# Reset loop and hope the problem has passed
if inputs_to_merge:
files_to_merge = []
for entries, file in inputs_to_merge:
files_to_merge.append(file)
self.merge_into_output(files_to_merge)
if done:
return
import subprocess
import tempfile
import minion.sensing.exceptions
import minion.sensing.postprocessors
import minion.core.components.exceptions
import multiprocessing
logger = multiprocessing.get_logger()
class PocketsphinxSpeechToText(minion.sensing.postprocessors.BasePostprocessor
):
configuration = {
'delete_audio_file':
True # Set this to false to debug by keeping audio file after request
}
def __init__(self, name, configuration={}):
super(PocketsphinxSpeechToText, self).__init__(name, configuration)
options = self.get_configuration('options', {})
self.options = ' '.join([
'-{} {}'.format(key, value) for key, value in options.iteritems()
])
def process(self, data):
with tempfile.NamedTemporaryFile(
delete=self.get_configuration('delete_audio_file')) as f:
f.write(data)
logger.debug('Writing to temporary file %s', f.name)
if not self.get_configuration('delete_audio_file'):
def synchronous_pull_with_lease_management(project_id, subscription_id):
"""Pulling messages synchronously with lease management"""
# [START pubsub_subscriber_sync_pull_with_lease]
import logging
import multiprocessing
import sys
import time
from google.api_core import retry
from google.cloud import pubsub_v1
multiprocessing.log_to_stderr()
logger = multiprocessing.get_logger()
logger.setLevel(logging.INFO)
processes = dict()
# TODO(developer)
# project_id = "your-project-id"
# subscription_id = "your-subscription-id"
subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(project_id,
subscription_id)
response = subscriber.pull(
request={
"subscription": subscription_path,
"max_messages": 3
},
retry=retry.Retry(deadline=300),
)
# Start a process for each message based on its size modulo 10.
for message in response.received_messages:
process = multiprocessing.Process(target=time.sleep,
args=(sys.getsizeof(message) % 10, ))
processes[process] = (message.ack_id, message.message.data)
process.start()
while processes:
# Take a break every second.
if processes:
time.sleep(1)
for process in list(processes):
ack_id, msg_data = processes[process]
# If the process is running, reset the ack deadline.
if process.is_alive():
subscriber.modify_ack_deadline(
request={
"subscription": subscription_path,
"ack_ids": [ack_id],
# Must be between 10 and 600.
"ack_deadline_seconds": 15,
})
logger.info(f"Reset ack deadline for {msg_data}.")
# If the process is complete, acknowledge the message.
else:
subscriber.acknowledge(request={
"subscription": subscription_path,
"ack_ids": [ack_id]
})
logger.info(f"Acknowledged {msg_data}.")
processes.pop(process)
print(
f"Received and acknowledged {len(response.received_messages)} messages from {subscription_path}."
)
# Close the underlying gPRC channel. Alternatively, wrap subscriber in
# a 'with' block to automatically call close() when done.
subscriber.close()
def error(*args):
"""Error function"""
return multiprocessing.get_logger().error(*args)
"""
Module with functions for dispatching subprocess and managing multiprocess pools
"""
import multiprocessing as mp
import subprocess
import os
import json
import logging
import argparse
import datetime
# grab logger from multiprocessing package
logger = mp.get_logger()
# setup logging levels
LOGGING_LEVELS = dict(
debug=logging.DEBUG,
info=logging.INFO,
warning=logging.WARNING,
error=logging.ERROR,
)
def subprocess_command(command,
path=None,
shell=False,
env=None,
pipe=False,
timeout=None):
"""
Execute command in subprocess.
def handle(self, *args, **options):
multiprocessing.log_to_stderr()
logger = multiprocessing.get_logger()
logger.setLevel(logging.INFO)
if PARALLEL:
pool = multiprocessing.Pool()
corpus_results = {}
results = []
total_l1 = []
total_l2 = []
filenames = []
for collection_root in COLLECTION_ROOTS: #iterate over phonbank and CHILDES
for collection_name in next(os.walk(collection_root))[1]:
print("*** ", collection_name)
corpus_roots = []
for root, dirnames, filenames in os.walk(
os.path.join(collection_root, collection_name)):
for filename in fnmatch.filter(filenames,
'*.zip_placeholder'):
corpus_roots.append(
os.path.join(
collection_root, collection_name, root,
filename.replace('.zip_placeholder', '')))
corpus_roots = list(set(corpus_roots))
print('Corpora to process: ' + str(len(corpus_roots)))
for corpus_root in corpus_roots:
if PARALLEL:
results.append(
pool.apply_async(count_corpora,
args=(corpus_root, )))
else:
results.append(count_corpora(corpus_root, ))
if PARALLEL:
pool.close()
for result in results:
try:
t1, t2, missing, filename = result.get()
total_l1.extend(t1)
total_l2.extend(t2) # would need a map for special subsetting
filenames.extend(filename)
except:
traceback.print_exc()
counts = numpy.asarray([total_l1, total_l2])
numpy.savetxt("freq.csv",
counts.transpose(),
delimiter=",",
fmt='%.3e')
print('Number of transcripts in correlation:')
print(len(total_l1))
coeff = numpy.corrcoef(total_l1, total_l2)[0, 1]
print("Correlation coefficient (Pearson's r)")
print(coeff)
os.system("echo %s > word_frequency_correlation.txt" % coeff)
import pdb
pdb.set_trace()
def getLogger(loggerName,
fileName='',
fileMode='a',
fileLevel='DEBUG',
printLevel='WARN',
colours={},
logger_type='logging',
capture_warnings=True):
"""Utility that returns a logger object
Parameters
----------
loggerName: str
Name for the logger, usually just provide __name__
fileName: str or bool, optional
Name for the log file, if False no log file is generated
fileMode: str, optional, default: 'a'
Mode for opening log file (e.g. 'w' for write, 'a' for append).
fileLevel: {'DEBUG','INFO','WARN'}, optional, default: 'DEBUG'
Log level for FileHandler
printLevel: {'DEBUG','INFO','WARN'}, optional, default: 'WARN'
Log level for StreamHandler
colours: dict, optional, default: {}
Override default colours for ColourStreamHandler (see .logging.colourstreamhandler.ColourStreamHandler for more info)
Returns
-------
logger: logging.Logger
A logger object with FileHandler and StreamHandler as configured. Note the logger is a singleton.
"""
logger_levels = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARN': logging.WARN
}
if logger_type == 'logging':
logger = logging.getLogger(loggerName)
elif logger_type == 'multiprocessing':
import multiprocessing
logger = multiprocessing.get_logger()
else:
raise ValueError(
f'invalid logger_type {logger_type}. Must be "logging" or "multiprocessing"'
)
logging.captureWarnings(capture_warnings)
if logger.hasHandlers():
logger.handlers[:] = []
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s/%(processName)s - %(module)s/%(funcName)s - %(levelname)s - %(message)s'
)
if fileName:
if not fileLevel in logger_levels:
raise ValueError(
f"Provided invalid fileLevel '{fileLevel}'. Valid options: {logger_levels.keys()}"
)
fileHandler = logging.FileHandler(fileName, mode=fileMode)
fileHandler.setLevel(logger_levels[fileLevel])
fileHandler.setFormatter(formatter)
logger.addHandler(fileHandler)
if not printLevel in logger_levels:
raise ValueError(
f"Provided invalid printLevel '{printLevel}'. Valid options: {logger_levels.keys()}"
)
try:
import colorama
except ImportError:
streamHandler = logging.StreamHandler()
colour_support = False
else:
from .colourstreamhandler import ColourStreamHandler
colorama.init(autoreset=True)
streamHandler = ColourStreamHandler(colours)
colour_support = True
streamHandler.setLevel(printLevel)
logger.addHandler(streamHandler)
if colour_support:
logger.info('Logger initialized with colour support.')
else:
logger.info(
'Logger initialized without colour support. Failed to import colorama.'
)
return logger
def Attack(packetList, e):
logger = multiprocessing.get_logger()
attackSocket = socket.socket(
socket.AF_INET,
socket.SOCK_RAW,
socket.IPPROTO_TCP
)
attackSocket.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
while True:
e.wait()
while packetList:
packet = packetList.pop(0)
ipPacket = IPv4Packet()
ihl = 5
version = 4
ipPacket.SetIpVersionIhl((version << 4) + ihl)
ipPacket.SetIpLen(40)
ipPacket.SetIpId(random.randint(0, 32767))
ipPacket.SetIpTTL(255)
ipPacket.SetIpProtocol(socket.IPPROTO_TCP)
ipPacket.SetIpSrc(packet["ipDst"])
ipPacket.SetIpDst(packet["ipSrc"])
ipHeader = ipPacket.Encode()
ipPacket.SetIpSum(ipHeader)
tcpPacket = TCPPacket()
if options["action"] == "RESET":
tcpPacket.SetTcpDestPort(packet["tcpSrcPort"])
tcpPacket.SetTcpSrcPort(packet["tcpDestPort"])
tcpPacket.SetTcpSeq(packet["tcpAckSeq"])
tcpPacket.SetTcpDoffReserved((5 << 4) + 0)
tcpPacket.SetTcpRstBit(1)
tcpPacket.SetTcpFlags()
logger.info("Attacking {}:{} and {}:{} with {}.".format(
packet["ipSrc"], packet['tcpSrcPort'],
packet["ipDst"], packet['tcpDestPort'], options["action"]))
tcpHeader = tcpPacket.Encode()
ipSrc = socket.inet_aton(str(packet["ipDst"]))
ipDst = socket.inet_aton(str(packet["ipSrc"]))
placeholder = 0
protocol = socket.IPPROTO_TCP
tcpLength = len(tcpHeader)
psudoHeader = pack('!4s4sBBH', ipSrc, ipDst, placeholder, protocol, tcpLength)
psudoHeader += tcpHeader
tcpPacket.SetTcpCheck(psudoHeader)
tcpHeader = tcpPacket.Encode()
result = ipHeader + tcpHeader
attackSocket.sendto(result, (str(packet["ipSrc"]), 0))
e.clear()
attackSocket.close()
def lat_opt_unpack(args):
import multiprocessing
logger = multiprocessing.get_logger()
args[2]['ihnf'] = args[3]
return args[3], lat_opt(args[0], args[1], **(args[2]))
def error(msg, *args):
return multiprocessing.get_logger().error(msg, *args)
def modify_insert_type_read(cigar,read,del_parts,pos,blocks,del_blocks,chr,shift_start,shift_end,HG):
logger=multiprocessing.get_logger()
read_parts=list()
del_part=list()
D_index=list()
I_index=list()
M_index=list()
S_index=list()
cigar_pat=re.compile(r'[0-9]+D|[0-9]+M|[0-9]+S|[0-9]+I')
D_M_array=cigar_pat.findall(cigar)
DM_len=list()
for item in D_M_array:
DM_len.append(int(item[:-1]))
sum=copy.deepcopy(DM_len)
sum[0]=DM_len[0]
for i in range(1,len(DM_len)):
if("D" in D_M_array[i]):
sum[i]=sum[i-1]
else:
sum[i]=sum[i-1]+DM_len[i]
for i in range(len(D_M_array)):
if "D" in D_M_array[i]:
D_index.append(i)
elif("I" in D_M_array[i]):
I_index.append(i)
elif("M" in D_M_array[i]):
M_index.append(i)
elif("S" in D_M_array[i]):
S_index.append(i)
cursor=0
for i in range(len(D_M_array)):
read_parts.append(read[cursor:sum[i]])
cursor=sum[i]
for i in range(len(D_M_array)):
if("D" in D_M_array[i]):
read_parts[i]=del_parts[0]
if(len(del_parts[1:])>0):
del_parts=del_parts[1:]
else:
break
Insert_l_r_blocks=dict()
M_match_blocks=dict()
M_block_index=0
for i in range(len(D_M_array)):
if("M" in D_M_array[i]):
M_match_blocks[i]=blocks[M_block_index]
M_block_index+=1
for i in range(len(D_M_array)):
if("I" in D_M_array[i]):
if(i>0 and i<(len(D_M_array)-1)):
'''
print 'D_M_array=',D_M_array
print 'i=',i
print 'len(M_match_blocks)=',len(M_match_blocks)
print 'cigar=',cigar
'''
Insert_l_r_blocks[i]=[M_match_blocks[i-1],M_match_blocks[i+1]]
Insert_delete_bool=dict()
for i in Insert_l_r_blocks:
pos_=Insert_l_r_blocks[i][0][1]
if (pos_>=shift_start and pos_<=shift_end):
if pos_!=pos:
read_parts[i]=""
else:
read_parts[i]=""
new_read_start,new_read_end=blocks[0][0],blocks[-1][1]
if(M_index[0]>0):
for i in range(M_index[0])[::-1]:
if("S" in D_M_array[i]):
if(len(read_parts[i])>10):
S_len=DM_len[i]
S_add_seq=get_ref(chr,new_read_start-S_len,new_read_start,HG)
read_parts[i]=S_add_seq
new_read_start-=len(read_parts[i])
if(M_index[-1]!=len(D_M_array)-1):
for i in range(M_index[-1]+1,len(D_M_array)):
if("S" in D_M_array[i]):
if(len(read_parts[i])>10):
S_len=DM_len[i]
S_add_seq=get_ref(chr,new_read_end,new_read_end+S_len,HG)
read_parts[i]=S_add_seq
new_read_end+=len(read_parts[i])
new_read="".join(read_parts)
return new_read,new_read_start,new_read_end
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import logging, multiprocessing
__all__ = ['FastTripsLogger', 'setupLogging']
#: This is the instance of :py:class:`Logger` that gets used for all dta logging needs!
FastTripsLogger = multiprocessing.get_logger()
def setupLogging(infoLogFilename,
debugLogFilename,
logToConsole=True,
append=False):
"""
Sets up the logger.
:param infoLogFilename: info log file, will receive log messages of level INFO or more important.
Pass None for no info log file.
:param debugLogFilename: debug log file, will receive log messages of level DEBUG or more important.
Pass None for no debug log file.
:param logToConsole: if true, INFO and above will also go to the console.
"""
def error(msg, *args):
""" Error function"""
return multiprocessing.get_logger(LOG_NAME).error(msg, *args)
def func_wrapper(test, flags):
logger = multiprocessing.get_logger()
logger.info("start: >{0} {1}".format(test, flags))
result = fn(test, flags)
logger.info("end: >{0} {1}".format(test, flags))
return result
import pylator as pyl
import multiprocessing as mp
import numpy as np
import time
from collections import deque
crit = mp.get_logger().critical
info = mp.get_logger().info
debug = mp.get_logger().debug
def module_configuration():
simData = {
# Define Module Outputs
"/outputs/signal/dtype": "double",
"/outputs/signal/shape": [200],
"/outputs/signal/default": 0,
# Define module inputs with assigned defaults
"/inputs/signal": 0
}
return simData
class Module(pyl.Model):
def initialise(self, simData):
# Inputs using default values, set as constants (Generated)
# User data
self.buffer = deque()
for i in range(simData["/outputs/signal/shape"][0] * 2):
self.buffer.append(0)
"""
import os, sys
import time
import json
import logging
import traceback
from optparse import OptionParser
from logging.handlers import RotatingFileHandler
from multiprocessing import get_logger, log_to_stderr
from ircbot import SingleServerIRCBot
from irclib import nm_to_n, all_events
log = get_logger()
_ourPath = os.getcwd()
_ourName = os.path.splitext(os.path.basename(sys.argv[0]))[0]
def relativeDelta(td):
s = ''
if td.days < 0:
t = "%s ago"
else:
t = "in %s"
days = abs(td.days)
seconds = abs(td.seconds)
minutes = seconds / 60
hours = minutes / 60
import multiprocessing as mp
from functools import partial
log = mp.get_logger()
def log_to_stderr(log_level='INFO', force=False):
"""
Shortcut allowing to display logs from workers.
:param log_level: Set the logging level of this logger.
:param force: Add handler even there are other handlers already.
"""
if not log.handlers or force:
mp.log_to_stderr()
log.setLevel(log_level)
def delayed(func):
"""
Decorator used to capture the arguments of a function.
Analogue of joblib's delayed.
:param func: Function to be captured.
"""
def wrapper(*args, **kwargs):
return partial(func, *args, **kwargs)
return wrapper
def __init__(self):
self._log = multiprocessing.get_logger()
self._lock = multiprocessing.Lock()
def asyncPoolEC(func, args, kwargs, NP=1, ldebug=False, ltrialnerror=True):
'''
A function that executes func with arguments args (len(args) times) on NP number of processors;
args must be a list of argument tuples; kwargs are keyword arguments to func, which do not change
between calls.
Func is assumed to take a keyword argument lparallel to indicate parallel execution, and return
a common exit status (0 = no error, > 0 for an error code).
This function returns the number of failures as the exit code.
'''
# input checking
if not isinstance(func, types.FunctionType): raise TypeError
if not isinstance(args, list): raise TypeError
if not isinstance(kwargs, dict): raise TypeError
if NP is not None and not isinstance(NP, int): raise TypeError
if not isinstance(ldebug, (bool, np.bool)): raise TypeError
if not isinstance(ltrialnerror, (bool, np.bool)): raise TypeError
# figure out if running parallel
if NP is not None and NP == 1: lparallel = False
else: lparallel = True
kwargs['ldebug'] = ldebug
kwargs['lparallel'] = lparallel
# logging level
if ldebug: loglevel = logging.DEBUG
else: loglevel = logging.INFO
# set up parallel logging (multiprocessing)
if lparallel:
multiprocessing.log_to_stderr()
mplogger = multiprocessing.get_logger()
#if ldebug: mplogger.setLevel(logging.DEBUG)
if ldebug: mplogger.setLevel(logging.INFO)
else: mplogger.setLevel(logging.ERROR)
# set up general logging
logger = logging.getLogger('multiprocess.asyncPoolEC') # standard logger
logger.setLevel(loglevel)
ch = logging.StreamHandler(sys.stdout) # stdout, not stderr
ch.setLevel(loglevel)
ch.setFormatter(logging.Formatter('%(message)s'))
logger.addHandler(ch)
kwargs['logger'] = logger.name
# # process sub logger
# sublogger = logging.getLogger('multiprocess.asyncPoolEC.func') # standard logger
# sublogger.propagate = False # only print message in this logger
# sch = logging.StreamHandler(sys.stdout) # stdout, not stderr
# sch.setLevel(loglevel)
# if lparallel: fmt = logging.Formatter('[%(processName)s] %(message)s')
# else: fmt = logging.Formatter('%(message)s')
# sch.setFormatter(fmt)
# sublogger.addHandler(sch)
# kwargs['logger'] = sublogger.name
# apply decorator
if ltrialnerror: func = TrialNError(func)
# print first logging message
logger.info(datetime.today())
logger.info('\nTHREADS: {0:s}, DEBUG: {1:s}\n'.format(
str(NP), str(ldebug)))
exitcodes = [] # list of results
def callbackEC(result):
# custom callback function that appends the results to the list
exitcodes.append(result)
## loop over and process all job sets
if lparallel:
# create pool of workers
pool = multiprocessing.Pool(
processes=NP) # NP=None uses all available CPUs
# distribute tasks to workers
for arguments in args:
#exitcodes.append(pool.apply_async(func, arguments, kwargs))
#print arguments
pool.apply_async(func, arguments, kwargs, callback=callbackEC)
# N.B.: we do not record result objects, since we have callback, which just extracts the exitcodes
# wait until pool and queue finish
pool.close()
pool.join()
logger.debug('\n *** all processes joined *** \n')
else:
# don't parallelize, if there is only one process: just loop over files
for arguments in args:
exitcodes.append(func(*arguments, **kwargs))
# evaluate exit codes
exitcode = 0
for ec in exitcodes:
#if lparallel: ec = ec.get() # not necessary, if callback is used
if ec < 0: raise ValueError, 'Exit codes have to be zero or positive!'
elif ec > 0: ec = 1
# else ec = 0, i.e. no errors
exitcode += ec
# N.B.: returnign None is interpreted as
nop = len(args) - exitcode
# print summary (to log)
if exitcode == 0:
logger.info(
'\n >>> All {:d} operations completed successfully!!! <<< \n'
.format(nop))
else:
logger.info(
'\n === {:2d} operations completed successfully! === \n'.
format(nop) +
'\n ### {:2d} operations did not complete/failed! ### \n'.
format(exitcode))
logger.info(datetime.today())
# return with exit code
return exitcode
