def process_2d(
self,
extractor,
glob_pattern="*.mp4",
timestamp_regex=TIMESTAMP_REGEX,
display_progress=True,
draw_viz=True,
):
log = Logger("App")
vi = VideoIndex.create_from_search(glob_pattern=glob_pattern, timestamp_regex=timestamp_regex)
log.info("Files to process: {}".format(vi.video_files))
length = len(vi.video_files)
processed_videos = []
files_by_timestamp = list(vi.groupby_iter("timestamp"))
disable_tqdm = not display_progress
with tqdm(files_by_timestamp, disable=disable_tqdm) as tqdm_files:
for timestamp, video_files in tqdm_files:
pvs = []
display_timestamp = timestamp.isoformat()
tqdm_files.set_description(display_timestamp)
with tqdm(video_files, disable=disable_tqdm) as tqdm_video_file:
for video_file in tqdm_video_file:
display_filename = "{} : {}".format(video_file.camera, video_file.file)
tqdm_video_file.set_description(display_filename)
pv = ProcessedVideo.from_video(
video_file.file,
extractor,
read_from_cache=False,
timestamp_start=timestamp,
draw_viz=draw_viz,
)
pvs.append(pv)
processed_videos.append((timestamp, pvs))
return processed_videos
def dispatch(args):
"""
Create a sample JSON file for a run, and launch it using Popen.
"""
dispatch_log = Logger('dispatch')
# validate that programs in setup file exist
setup_params = json.load(args.setup)
validate_setupfile(setup_params)
# validate reference
validate_reference(args.ref)
# validate directories
validate_directory(args.log, dispatch_log)
validate_directory(args.stats, dispatch_log)
validate_directory(args.bam_dir, dispatch_log)
if (args.scythe and args.adapter is None) or (args.adapter is not None and not os.path.isfile(args.adapter)):
logger.critical("adapter file for Scythe no specified, or does not exist.")
sys.exit(1)
# create sample config JSON file, starting off with global config passed through args
global_sample_config = dict(reference=args.ref, adapters_file=args.adapter,
prior=str(args.prior), error=args.trim_error,
stats_dir=args.stats, nthreads=args.threads,
mem=args.mem, bam_dir=args.bam_dir)
# which preprocess steps to use
global_sample_config["preprocess-steps"] = list()
for step in PREPROCESSING_STEPS:
if step in args and args.__getattribute__(step):
global_sample_config["preprocess-steps"].append(step)
global_params = dict(global_sample_config.items() + setup_params.items())
sample_config = "%s_samples.txt" % args.job
samples = create_sample_config(args.samples, sample_config, global_params)
# create batch script
sbatch_params = {"log_dir":args.log, "jobname":args.job, "nthreads":args.threads,
"mem":args.mem, "nsamples":len(samples), "sample_dispatch_py":__file__,
"sample_config":sample_config, "partition":args.partition}
batch_script = safe_templater(SLURM_BATCH, sbatch_params)
batch_file = "%s_batch.sh" % args.job
with open(batch_file, 'w') as f:
f.write(batch_script)
if not args.dry_run:
# now, start the batch script
dispatch_log.info("submitting sbatch script '%s'." % batch_file)
sbatch_cmd = ["sbatch"]
if args.email is not None:
sbatch_cmd.extend(["--mail-type", "ALL"])
sbatch_cmd.extend(["--mail-user", args.email])
sbatch_cmd.append(batch_file)
retcode = check_call(sbatch_cmd)
if retcode != 0:
dispatch_log.critical("submitting batch script '%s' exited abnormally with return code %d." % (batch_file, retcode))
sys.exit(retcode.returncode)
dispatch_log.info("submitting sbatch script '%s' complete." % batch_file)
def pds_database( log: Logger,
filter: Optional[Union[str, Path]] = None,
) -> Tuple[Path, List]:
"""Provide the list of target PDS as a file and a list.
.. note::
Depends on the ``master.pds`` configuration option.
:param log: Job logger.
:param filter: File containting the target subset, otherwise all PDS database is considered.
"""
# @TODO: PDS-FILTER
pds_file = Path(TBcore.get_option('master', 'pds'))
if pds_file.is_file():
pds_list = [line.strip() for line in open(pds_file).readlines() if len(line.strip()) > 0]
elif pds_file.is_dir():
pds_list = [str(x.resolve()) for x in pds_file.glob('*/*.pds')]
else:
raise NodeDataError('The provided MASTER database directory/list file cannot be found.')
# Even if a PDS file already exists, we always create a temporary file so that we can
# manage different versions of the PDS database in different Nodes.
f = NamedTemporaryFile(mode='w', delete=False)
log.info(f'Temporary file for PDS database: {f.name}')
[f.write(x + '\n') for x in pds_list]
f.close()
pds_file = Path(f.name)
return pds_file, pds_list
class Aiko(commands.AutoShardedBot):
"""
Aiko's bot class.
Subclass of discord.ext.commands.AutoShardedBot.
See discord.ext.commands documentation for parameters
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.config = Config("config.yaml").config
StreamHandler(sys.stdout).push_application()
self.log = Logger("Aiko")
self.osuapi = OsuApi(self.config["osuapi"], connector=AHConnector())
self.kv = self.loop.run_until_complete(_connect_redis(self))
async def on_ready(self):
self.log.info(f"Aiko is ready! {len(self.guilds)} servers")
await self.change_presence(
activity=discord.Streaming(
name=f"osu! // {get_prefix(self.config)}help",
url="https://twitch.tv/monstercat",
)
)
async def on_message(self, message):
if message.author.bot:
return
ctx = await self.get_context(message)
await self.invoke(ctx)
class actionThread(stopableThread):
def __init__(self,actList, inputParam=None,*args,**kargs):
super(actionThread,self).__init__(*args,**kargs)
self.actList=actList
self.param = inputParam
self.log = Logger(u"%s actionThread"%self.name)
def setParam(self,data):
self.param = data
def changeActList(self,newList):
if type(newList) != type(()) and type(newList) != type(()):
self.log.error(u"changeActList:invalid parameter")
return
self.actList = newList
def process(self):
"""
调用actList中的所有action plugin.Only run once.
"""
actManager = absManager.actionManager()
for i in self.actList:
self.log.info(u"do action %s:%s" % (i,actManager.id2filename(i)))
actManager.call(i,self.param)
self.pause()
def error(self, id_=None, error_code=None, error_msg=None):
if isinstance(id_, Exception):
# XXX: for an unknown reason 'log' is None in this branch,
# therefore it needs to be instantiated before use
global log
if not log:
log = Logger('IB Broker')
log.exception(id_)
if isinstance(error_code, EClientErrors.CodeMsgPair):
error_msg = error_code.msg()
error_code = error_code.code()
if isinstance(error_code, int):
if error_code in (502, 503, 326):
# 502: Couldn't connect to TWS.
# 503: The TWS is out of date and must be upgraded.
# 326: Unable connect as the client id is already in use.
self.unrecoverable_error = True
if error_code < 1000:
log.error("[{}] {} ({})".format(error_code, error_msg, id_))
else:
log.info("[{}] {} ({})".format(error_code, error_msg, id_))
else:
log.error("[{}] {} ({})".format(error_code, error_msg, id_))
def runner(args):
"""
Run a sample through an NGS pipeline (as a command) using Popen.
TOOD: logging, timing.
"""
if args.config is None:
sample_params = json.loads(sys.stdin.readline().rstrip())
else:
# read the first line from the test/debug config file
sample_params = json.loads(args.config.readline().rstrip())
sample = sample_params["sample_id"]
runner_log = Logger("%s logger" % sample)
cmd = safe_templater(SAMPLE_PROCESS_CMD, sample_params)
runner_log.info("%s starting preprocessing and alignment of sample." % sample)
if args.dry_run:
runner_log.debug("%s command: %s" % (sample, cmd))
return
tstart = time.time()
p = Popen(cmd, shell=True, executable=find_bash())
p.wait()
if p.returncode != 0:
# make this as loud as possible so Slurm can handle it
runner_log.critical("%s exited abnormally with return code %d." % (sample, p.returncode))
sys.exit(p.returncode)
tend = time.time()
elapsed = tend - tstart
runner_log.info("%s completed preprocessing and alignment in %s seconds." % (sample, str(round(elapsed, 5))))
class CustomLogger(object):
def __init__(
self,
log_level=LogLevel.INFO,
format_str='[{record.time:%Y-%m-%d %H:%M:%S}] - {record.channel} - {record.level_name} '
'- {record.message}'):
self.logger = Logger('WindAdapter')
set_datetime_format('local')
StreamHandler(sys.stdout, format_string=format_str).push_application()
FileHandler('WindAdapter.log', bubble=True,
format_string=format_str).push_application()
self.set_level(log_level)
def set_level(self, log_level):
if log_level.lower() == LogLevel.INFO:
self.logger.level = logbook.INFO
elif log_level.lower() == LogLevel.WARNING:
self.logger.level = logbook.WARNING
elif log_level.lower() == LogLevel.CRITICAL:
self.logger.level = logbook.CRITICAL
elif log_level.lower() == LogLevel.NOTSET:
self.logger.level = logbook.NOTSET
def info(self, msg):
self.logger.info(msg)
def warning(self, msg):
self.logger.warning(msg)
def critical(self, msg):
self.logger.critical(msg)
def error(self, id_=None, error_code=None, error_msg=None):
if isinstance(id_, Exception):
# XXX: for an unknown reason 'log' is None in this branch,
# therefore it needs to be instantiated before use
global log
if not log:
log = Logger('IB Broker')
log.exception(id_)
if isinstance(error_code, EClientErrors.CodeMsgPair):
error_msg = error_code.msg()
error_code = error_code.code()
if isinstance(error_code, int):
if error_code in (502, 503, 326):
# 502: Couldn't connect to TWS.
# 503: The TWS is out of date and must be upgraded.
# 326: Unable connect as the client id is already in use.
self.unrecoverable_error = True
if error_code < 1000:
log.error("[{}] {} ({})".format(error_code, error_msg, id_))
else:
log.info("[{}] {} ({})".format(error_code, error_msg, id_))
else:
log.error("[{}] {} ({})".format(error_code, error_msg, id_))
def start_parse(debug, file, save):
""" main function for starting everything"""
if debug:
log_level = 'DEBUG'
else:
log_level = 'WARNING'
StreamHandler(stdout, level=log_level).push_application()
log = Logger('main')
log.debug('Starting up...')
games = parse(file)
log.info("Games:", games.count)
if save:
log.debug('Saving data to json files')
l_game = []
for game in games.items():
name = game.sid.strftime("%Y-%m-%d_%H%M%S")
l_game.append(name)
filename = save + name + ".json"
with open(filename, 'w') as file:
file.write(jsonpickle.encode(game))
log.debug("wrote game info to {file}".format(file=filename))
filename = save + 'hs2017.json'
highscore = score_count(games)
data = {'games': l_game, 'highscore': highscore}
with open(filename, 'w') as file:
log.debug("wrote highscore info to {file}".format(file=filename))
file.write(json.dumps(data))
log.debug('End of program...')
def runner(args):
"""
Run a sample through an NGS pipeline (as a command) using Popen.
TOOD: logging, timing.
"""
if args.config is None:
sample_params = json.loads(sys.stdin.readline().rstrip())
else:
# read the first line from the test/debug config file
sample_params = json.loads(args.config.readline().rstrip())
sample = sample_params["sample_id"]
runner_log = Logger("%s logger" % sample)
cmd = safe_templater(SAMPLE_PROCESS_CMD, sample_params)
runner_log.info("%s starting preprocessing and alignment of sample." %
sample)
if args.dry_run:
runner_log.debug("%s command: %s" % (sample, cmd))
return
tstart = time.time()
p = Popen(cmd, shell=True, executable=find_bash())
p.wait()
if p.returncode != 0:
# make this as loud as possible so Slurm can handle it
runner_log.critical("%s exited abnormally with return code %d." %
(sample, p.returncode))
sys.exit(p.returncode)
tend = time.time()
elapsed = tend - tstart
runner_log.info("%s completed preprocessing and alignment in %s seconds." %
(sample, str(round(elapsed, 5))))
def runner(args):
"""
Run a sample through an NGS pipeline (as a command) using Popen.
TOOD: logging, timing.
"""
if args.config is None:
sample_params = json.loads(sys.stdin.readline().rstrip())
else:
# read the first line from the test/debug config file
sample_params = json.loads(args.config.readline().rstrip())
sample = sample_params["sample_id"]
runner_log = Logger("%s logger" % sample)
tstart = datetime.datetime.now() # total run time
# preprocessing and alignment
aln_cmd = build_sample_aln_command(sample_params)
run_command_on_sample(aln_cmd, runner_log, sample, desc="preprocessing and alignment")
if args.dry_run:
return
sort_cmd = safe_templater(SORT_STEPS['samtools-sort'], sample_params)
run_command_on_sample(sort_cmd, runner_log, sample, desc="sorting BAM file")
tend = datetime.datetime.now()
elapsed = tend - tstart
runner_log.info("%s all processing completed in: %s." % (sample, str(elapsed)))
class Router(Thread):
"""Thread waiting for a request by another Driver and responding to
it with the chunked asked.
"""
def __init__(self, name, redis, get_chunk):
super(Router, self).__init__()
self.name = name
self.redis = redis
self.get_chunk = get_chunk
self.router = None
self.logger = Logger("{} - Router".format(self.name))
self.logger.info("Started")
self.context = zmq.Context.instance()
def run(self):
self.router = self.context.socket(zmq.ROUTER)
port = self.router.bind_to_random_port('tcp://*')
self.redis.set('drivers:{}:router'.format(self.name), port)
while True:
msg = self.router.recv_multipart()
self._respond_to(*msg)
def _respond_to(self, identity, filename, offset, size):
"""Calls the `get_chunk` handler defined by the Driver to get
the chunk and send it to the addressee.
"""
self.logger.debug("Getting chunk of size {} from offset {} in '{}'",
size, offset, filename)
chunk = self.get_chunk(filename, int(offset), int(size))
self.router.send_multipart((identity, chunk))
def checkpoint_out( log: Logger, filename: Union[Path, str], data: Dict ):
"""Dump checkpoint.
"""
filename = Path(filename)
log.info(f'CHECKPOINT: Creating at {filename}\n')
with filename.open('w') as fd:
json.dump(data, fd, cls=GeneralEncoder)
def write_rating_to_bcolz():
table_name = 'rating'
logger = Logger(table_name)
logger.info('准备数据......')
df = rating_table()
attr_dict = {}
ndays = 0
write_dataframe(df, table_name, ndays, attr_dict)
def test2():
log = Logger('Logbook-test-2')
log.critical("critical")
log.error("error")
log.warn("warn")
log.notice("notice")
log.info("test")
log.debug("debug")
def write_static_info_to_bcolz():
table_name = 'infoes'
logger = Logger(table_name)
logger.info('准备数据......')
ndays = 0
df, attr_dict = static_info_table()
# df['asof_date'] = pd.to_datetime(df['asof_date'].values).tz_localize(None)
write_dataframe(df, table_name, ndays, attr_dict)
def dispatch(args):
"""
Create a sample JSON file for a run, and launch it using Popen.
"""
dispatch_log = Logger("dispatch")
# validate that programs in setup file exist
setup_params = json.load(args.setup)
validate_setupfile(setup_params)
# validate reference
validate_reference(args.ref)
# validate directories
validate_directory(args.log, dispatch_log)
validate_directory(args.stats, dispatch_log)
validate_directory(args.bam_dir, dispatch_log)
# create sample config JSON file, starting off with global config passed through args
global_sample_config = dict(
reference=args.ref,
adapters_file=args.adapter,
prior=str(args.prior),
error=args.trim_error,
stats_dir=args.stats,
nthreads=args.threads,
mem=args.mem,
bam_dir=args.bam_dir,
)
global_params = dict(global_sample_config.items() + setup_params.items())
sample_config = "%s_samples.txt" % args.job
samples = create_sample_config(args.samples, sample_config, global_params)
# create batch script
sbatch_params = {
"log_dir": args.log,
"jobname": args.job,
"nthreads": args.threads,
"mem": args.mem,
"nsamples": len(samples),
"sample_dispatch_py": __file__,
"sample_config": sample_config,
"partition": args.partition,
}
batch_script = safe_templater(SLURM_BATCH, sbatch_params)
batch_file = "%s_batch.sh" % args.job
with open(batch_file, "w") as f:
f.write(batch_script)
if not args.dry_run:
# now, start the batch script
dispatch_log.info("submitting sbatch script '%s'." % batch_file)
retcode = check_call(["sbatch", batch_file])
if retcode != 0:
dispatch_log.critical(
"submitting batch script '%s' exited abnormally with return code %d." % (batch_file, retcode)
)
sys.exit(retcode.returncode)
dispatch_log.critical("submitting sbatch script '%s' complete." % batch_file)
def dispatch(args):
"""
Create a sample JSON file for a run, and launch it using Popen.
"""
dispatch_log = Logger('dispatch')
# validate that programs in setup file exist
setup_params = json.load(args.setup)
validate_setupfile(setup_params)
# validate reference
validate_reference(args.ref)
# validate directories
validate_directory(args.log, dispatch_log)
validate_directory(args.stats, dispatch_log)
validate_directory(args.bam_dir, dispatch_log)
# create sample config JSON file, starting off with global config passed through args
global_sample_config = dict(reference=args.ref,
adapters_file=args.adapter,
prior=str(args.prior),
error=args.trim_error,
stats_dir=args.stats,
nthreads=args.threads,
mem=args.mem,
bam_dir=args.bam_dir)
global_params = dict(global_sample_config.items() + setup_params.items())
sample_config = "%s_samples.txt" % args.job
samples = create_sample_config(args.samples, sample_config, global_params)
# create batch script
sbatch_params = {
"log_dir": args.log,
"jobname": args.job,
"nthreads": args.threads,
"mem": args.mem,
"nsamples": len(samples),
"sample_dispatch_py": __file__,
"sample_config": sample_config,
"partition": args.partition
}
batch_script = safe_templater(SLURM_BATCH, sbatch_params)
batch_file = "%s_batch.sh" % args.job
with open(batch_file, 'w') as f:
f.write(batch_script)
if not args.dry_run:
# now, start the batch script
dispatch_log.info("submitting sbatch script '%s'." % batch_file)
retcode = check_call(["sbatch", batch_file])
if retcode != 0:
dispatch_log.critical(
"submitting batch script '%s' exited abnormally with return code %d."
% (batch_file, retcode))
sys.exit(retcode.returncode)
dispatch_log.critical("submitting sbatch script '%s' complete." %
batch_file)
def top_words(sorted_set):
log = Logger("Top Words")
top = []
while True:
if(len(sorted_set) > 0):
test = list(reversed(sorted_set[-10:]))
if 0 in [item in top for item in test]:
top = test
log.info('#1-10 of {}: {}'.format(len(sorted_set), ', '.join(top)))
time.sleep(0.01)
class Bot(commands.Bot):
def __init__(self, config):
super(Bot, self).__init__(command_prefix=self.COMMAND_PREFIX)
self.config = config
self.logger = Logger(self.__class__.__name__.lower())
self.event(self.on_ready)
self.add_cog(self.COMMANDS(self))
def run(self):
super(Bot, self).run(self.config.token)
async def on_ready(self):
self.logger.info(f'Logged in: {self.user.name} {self.user.id}')
def load_plugins(self) -> None:
"""Dynamically loads all plugins from the plugins directory.
New plugins can be added by creating new classes in the `plugins` module.
"""
self.plugins = {}
importlib.import_module("plugins")
modules = []
plugin_files = os.listdir(os.path.join(os.path.dirname(__file__), "plugins"))
if len(plugin_files) == 0:
print("NOTE: No plugin files found.")
for plugin in plugin_files:
if plugin.startswith("__") or not plugin.endswith(".py"):
# Skip files like __init__.py and .gitignore
continue
module_name = "plugins." + plugin.rsplit(".")[0]
modules.append(importlib.import_module(module_name, package="plugins"))
for module in modules:
if module.__name__ in sys.modules:
importlib.reload(module)
clsmembers = inspect.getmembers(
module,
lambda member: inspect.isclass(member)
and member.__module__ == module.__name__,
)
for name, cls in clsmembers:
if not issubclass(cls, BasePlugin):
# We only want plugins that derive from BasePlugin
CORE_LOG.warn(
f"Skipping {name} as it doesn't derive from the BasePlugin"
)
continue
CORE_LOG.info(f"Loading plugin {name} ...")
# Create logger for each plugin
plugin_logger = Logger(f"olive.plugin.{name}")
plugin_logger.info(f"{name}'s logger is working hard!")
logger_group.add_logger(plugin_logger)
# Generate standard config
config = PluginConfig(plugin_logger)
# Instantiate the plugin!
self.plugins[name] = cls(config)
CORE_LOG.info("Loaded plugins")
def write_dataframe(df, table_name, ndays, attr_dict):
"""以bcolz格式写入数据框"""
logger = Logger(table_name)
# 列名称及类型标准化
out = _normalize_ad_ts_sid(df, ndays=ndays)
# 转换为bcolz格式并存储
rootdir = bcolz_table_path(table_name)
if os.path.exists(rootdir):
rmtree(rootdir)
odo(out, rootdir)
logger.info('数据存储路径：{}'.format(rootdir))
# 设置属性
ct = bcolz.open(rootdir)
for k, v in attr_dict.items():
ct.attrs[k] = v
class timer(object):
"""Decorator that mesures the time it takes to run a function."""
__instances = {}
def __init__(self, f):
self.__f = f
self.log = Logger(f.func_name)
def __call__(self, *args, **kwargs):
self.__start = time.time()
result = self.__f(*args, **kwargs)
value = time.time() - self.__start
self.log.info('ellapsed time: {0:.2f}ms'.format(value * 1000))
return result
def sketchXZ(log: Logger, cases: List[Case],
**kwargs) -> Tuple[plt.Figure, List[plt.Axes]]:
"""
"""
grid = _calculate_grid(cases, **kwargs)
if TBcore.get_option('system', 'verbose'):
sys.stdout.write('Generating an image grid of: {0}x{1}\n'.format(
grid[0], grid[1]))
fsize = (kwargs.pop('width',
7.5 * grid[1]), kwargs.pop('hight', 7.5 * grid[0]))
fig = plt.figure(figsize=fsize)
axs = []
ylim, xlim = [0, 0], [0, 0]
for i, case in enumerate(cases):
position = (int(i / grid[1]), i % grid[1])
title = '{0}_{1:03d}'.format(case['configuration.name'], i + 1)
log.info(
f'Showing {title}-{case.architecture_str} in position: {position[0]}x{position[1]}\n'
)
ax = plt.subplot2grid(grid, position, fig=fig)
axs.append(ax)
plot_case_sketch(case, ax, kwargs.pop('connections', False),
kwargs.pop('beta_fill', 'red'),
kwargs.pop('beta_edge', 'black'),
kwargs.pop('alpha_fill', 'blue'),
kwargs.pop('alpha_edge', 'black'),
kwargs.pop('connection_edge', None))
ax.set_title(title)
cy = ax.get_ylim()
cx = ax.get_xlim()
ylim = [
ylim[0] if cy[0] < ylim[0] else cy[0],
ylim[1] if cy[1] > ylim[1] else cy[1]
]
xlim = [
xlim[0] if cx[0] > xlim[0] else cx[0],
xlim[1] if cx[1] < xlim[1] else cx[1]
]
for ax in axs:
ax.set_ylim(ylim[0], ylim[1])
ax.set_xlim(xlim[0], xlim[1])
return fig, axs
class PyPLogger(object):
def __init__(self, clazz):
logbook.set_datetime_format("local")
self.serverName = clazz.__name__[clazz.__name__.rfind('.') + 1:]
if not os.path.exists(log_dir):
os.makedirs(log_dir)
self.log_file = TimedRotatingFileHandler(os.path.join(
log_dir, '%s.log' % self.serverName),
date_format='%Y-%m-%d',
bubble=True,
encoding='utf-8')
self.log_std = ColorizedStderrHandler(bubble=True)
# self.log_std = StderrHandler(bubble=True)
self.log = Logger(self.serverName)
self.__init_logger()
self.__setting()
def log_type(self, record, handler):
# log = "[{date}]-[{level}]-[" + self.serverName + "] - {msg}".format(
log = "[" + self.serverName + "]" + "-[{date}]-[{level}] - {msg}".format(
date=record.time,
level=record.level_name,
# filename = os.path.split(record.filename)[-1],
# func_name = record.func_name,
# lineno = record.lineno,
msg=record.message)
return log
def __init_logger(self):
logbook.set_datetime_format("local")
self.log.handlers = []
self.log.handlers.append(self.log_file)
self.log.handlers.append(self.log_std)
def __setting(self):
self.log_std.formatter = self.log_type
self.log_file.formatter = self.log_type
def info(self, *args, **kwargs):
self.log.info(*args, **kwargs)
def warn(self, *args, **kwargs):
self.log.warn(*args, **kwargs)
def error(self, *args, **kwargs):
self.log.error(*args, **kwargs)
class Plugin(object):
def __init__(self, site, name=None):
self.name = name or remove_suffix('plugin', self.__class__.__name__)
self.log = Logger(self.__class__.__name__.lower())
self.log.debug('{} initialized'.format(self.name))
self.base_dir = Path(inspect.getfile(self.__class__)).parent
# initialize templates
template_path = self.base_dir / 'templates'
if template_path.exists():
self.jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(str(template_path)),
extensions=['jinja2.ext.with_']
)
# load possible default configuration
self.register(site)
@property
def DEFAULTS_FILE(self):
return self.base_dir / 'defaults.cfg'
def register(self, site):
pass
def enable_app(self, app):
pass
def render_template(self, template_name, **kwargs):
if not hasattr(self, 'jinja_env'):
return RuntimeError('Plugin {} has no template path'.format(
self.__class__.__name__
))
tpl = self.jinja_env.get_template(template_name)
return tpl.render(**kwargs)
def output_template(self, template_name, dest, _mode=0o644, **kwargs):
if not dest.parent.exists():
self.log.warning('Path {} did not exist and was created'.format(
dest.parent,
))
dest.parent.mkdir(parents=True)
with new_file(dest, _mode) as out:
self.log.info('Writing {}'.format(dest.resolve()))
out.write(self.render_template(template_name, **kwargs))
class fileMonitor(object):
"""
监视文件目录的变更
"""
def __init__(self,path):
super(fileMonitor,self).__init__()
self.log = Logger(u"fileMonitor")
self.event_handler = EventHandler()
self.observer = Observer()
self.observer.schedule(self.event_handler, path, recursive=True)
self.log.info("now starting monitor path:%s" % path)
def start(self):
self.observer.start()
def stop(self):
self.observer.stop()
self.observer.join()
def main():
global log
StderrHandler().push_application()
log = Logger("xbbs.coordinator")
XBBS_CFG_DIR = os.getenv("XBBS_CFG_DIR", "/etc/xbbs")
with open(path.join(XBBS_CFG_DIR, "coordinator.toml"), "r") as fcfg:
cfg = CONFIG_VALIDATOR.validate(toml.load(fcfg))
inst = Xbbs.create(cfg)
for name, elem in cfg["projects"].items():
project = Project(name,
**elem,
base=path.join(inst.project_base, name))
inst.projects[name] = project
os.makedirs(project.base, exist_ok=True)
log.debug("got project {}", inst.projects[name])
with inst.zmq.socket(zmq.REP) as sock_cmd, \
inst.zmq.socket(zmq.PULL) as inst.intake, \
inst.zmq.socket(zmq.ROUTER) as inst.worker_endpoint:
# XXX: potentially make perms overridable? is that useful in any
# capacity?
inst.intake.bind(cfg["intake"]["bind"])
_ipc_chmod(cfg["intake"]["bind"], 0o664)
inst.worker_endpoint.bind(cfg["worker_endpoint"])
inst.worker_endpoint.set(zmq.ROUTER_MANDATORY, 1)
_ipc_chmod(cfg["worker_endpoint"], 0o664)
sock_cmd.bind(cfg["command_endpoint"]["bind"])
_ipc_chmod(cfg["command_endpoint"]["bind"], 0o664)
dumper = gevent.signal_handler(signal.SIGUSR1, dump_projects, inst)
log.info("startup")
intake = gevent.spawn(intake_loop, inst)
job_pull = gevent.spawn(job_pull_loop, inst)
try:
command_loop(inst, sock_cmd)
finally:
# XXX: This may not be the greatest way to handle this
gevent.killall(inst.project_greenlets[:])
gevent.kill(intake)
gevent.kill(job_pull)
dumper.cancel()
def create_app(config_class = DefaultConfig):
app = connexion.FlaskApp(
__name__, specification_dir='openapi/', options={"swagger_ui": False, "serve_spec": False}
)
app.app.config.from_object(config_class)
log = Logger('logbook')
log.info(app.app.config['LOG_LEVEL'])
#show logging messages in terminal
StreamHandler(sys.stdout ,
level = app.app.config['LOG_LEVEL']).push_application()
log.info('welcome to my application CHALLENGE CODE API MODE {}'.format(env('FLASK_ENV','developement')))
app.add_api("swagger.yaml", strict_validation=True)
flask_app = app.app
return flask_app
def process_3d(
self,
cameras: typing.Mapping[str, ImmutableCamera],
glob_pattern="*.mp4",
timestamp_regex=TIMESTAMP_REGEX,
display_progress=True,
output_file_pattern="ProcessedVideo3D_{}.pb",
):
log = Logger("App")
vi = VideoIndex.create_from_search(glob_pattern=glob_pattern, timestamp_regex=TIMESTAMP_REGEX)
log.info("Files to process: {}".format(vi.video_files))
length = len(vi.video_files)
processed_videos = []
files_by_timestamp = list(vi.groupby_iter("timestamp"))
DATETIME_FORMAT = "%Y-%m-%d-%H-%M-%S"
disable_tqdm = not display_progress
with tqdm(files_by_timestamp, disable=disable_tqdm) as tqdm_files:
for timestamp, video_files in tqdm_files:
pvs = []
display_timestamp = timestamp.isoformat()
tqdm_files.set_description(display_timestamp)
with tqdm(video_files, disable=disable_tqdm) as tqdm_video_file:
cams = []
for video_file in tqdm_video_file:
display_filename = "{} : {}".format(video_file.camera, video_file.file)
tqdm_video_file.set_description(display_filename)
pv = ProcessedVideo.from_video(
video_file.file,
None,
read_from_cache=True,
write_to_cache=False,
timestamp_start=timestamp,
draw_viz=False,
)
pvs.append(pv)
cams.append(cameras[video_file.camera])
pv3d = ProcessedVideo3D.from_processed_video_2d(pv2ds=pvs, cameras=cams)
date_text = timestamp.strftime(DATETIME_FORMAT)
output_filename = output_file_pattern.format(date_text)
pv3d.to_proto_file(output_filename)
processed_videos.append((timestamp, pv3d))
return processed_videos
def checkpoint_in( log: Logger, filename: Union[Path, str] ) -> Optional[Dict]:
"""Incoming checkpoint.
"""
if TBcore.get_option('system', 'forced'):
return None
filename = Path(filename)
if filename.is_file():
log.info(f'CHECKPOINT: Reloading from {filename}\n')
with Path(filename).open() as fd:
try:
data = json.load(fd)
except json.JSONDecodeError:
return None
return data
return None
def write_dataframe(df, table_name, attr_dict=None):
"""以bcolz格式写入数据框"""
log = Logger(table_name)
# 转换为bcolz格式并存储
rootdir = bcolz_table_path(table_name)
if os.path.exists(rootdir):
rmtree(rootdir)
df = _normalize_ad_ts_sid(df)
for c in (AD_FIELD_NAME, TS_FIELD_NAME, SID_FIELD_NAME):
if df[c].hasnans:
warnings.warn(f'{c}列含有空值，已移除')
df = df.loc[~df[c].isnan(), :]
ct = bcolz.ctable.fromdataframe(df, rootdir=rootdir)
log.info('写入数据至：{}'.format(rootdir))
if attr_dict:
# 设置属性
for k, v in attr_dict.items():
ct.attrs[k] = v
class Root(resource.Resource):
isLeaf = True
def __init__(self, lock):
self._lock = lock
self.log = Logger('web')
resource.Resource.__init__(self)
@delayed
def render_GET(self, request):
try:
key = _get_key_from_path(request.path)
if key == 'info/keys':
request.write('%r\n' % (self._lock._keys,))
elif key == 'info/status':
for line in self._lock.get_status():
request.write('%s %s\n' % line)
elif key == 'info/log':
for line in self._lock._log:
request.write('%s\n' % line)
else:
value = yield self._lock.get_key(key)
request.write(value)
except KeyNotFound:
request.setResponseCode(NOT_FOUND)
returnValue('')
@proxy_to_master
@delayed
def render_POST(self, request):
try:
key = _get_key_from_path(request.path)
data = request.args.get('data', [''])[0]
self.log.info('Set key %s=%r' % (key, data))
yield self._lock.set_key(key, data)
except KeyAlreadyExists, e:
self.log.warning(e)
request.setResponseCode(CONFLICT)
except PaxosError, e:
self.log.warning(e)
request.setResponseCode(EXPECTATION_FAILED)
def main():
logbook.concurrency.enable_gevent()
global log
StderrHandler().push_application()
log = Logger('xbbs.worker')
inst = XbbsWorker()
XBBS_CFG_DIR = os.getenv("XBBS_CFG_DIR", "/etc/xbbs")
with open(path.join(XBBS_CFG_DIR, "worker.toml"), "r") as fcfg:
cfg = CONFIG_VALIDATOR.validate(toml.load(fcfg))
job_request = msgs.JobRequest(capabilities=cfg["capabilities"]).pack()
gevent.signal_handler(signal.SIGUSR1, gevent.util.print_run_info)
log.info(cfg)
while True:
with inst.zmq.socket(zmq.REQ) as jobs:
jobs.connect(cfg["job_endpoint"])
while True:
jobs.send(job_request)
log.debug("waiting for job...")
# the coordinator sends a heartbeat each minute, so 1.5 minutes
# should be a sane duration to assume coordinator death on
if jobs.poll(90000) == 0:
# breaking the inner loop will cause a reconnect
# since the coordinator is presumed dead, drop requests yet
# unsent to it
jobs.set(zmq.LINGER, 0)
log.debug("dropping socket after a heartbeat timeout")
break
try:
msg = jobs.recv()
if len(msg) == 0:
# drop null msgs
continue
process_job_msg(inst, msg)
except KeyboardInterrupt:
log.exception("interrupted")
return
except Exception as e:
log.exception("job error", e)
class DeleteFiles:
def __init__(self):
handler = TimedRotatingFileHandler('../logs/delete_files.log')
handler.push_application()
self.logger = Logger(name='delete files')
self.path_lists = self.get_conf()
@staticmethod
def get_conf():
##从配置文件获取配置
cp = ConfigParser.ConfigParser()
with codecs.open(os.path.join(BASE_DIR, './config/config.ini'),
'r',
encoding='utf-8') as f:
cp.readfp(f)
path_lists = eval(cp.get('filesystem', 'del_paths').strip())
return path_lists
def file_handler(self):
for file_path in self.path_lists:
file_name = os.path.join('/', file_path)
if os.path.isdir(file_name) or os.path.isfile(file_name):
if os.path.isdir(file_name):
try:
os.removedirs(file_name)
#os.remove(file_name)
except Exception, e:
self.logger.error('Delete dir %s failed!%s' %
(file_name, e))
else:
self.logger.info('Delete dir %s success!' % file_name)
if os.path.isfile(file_name):
try:
#os.removedirs(file_name)
os.remove(file_name)
except Exception, e:
self.logger.error('Delete file %s failed!%s' %
(file_name, e))
else:
self.logger.info('Delete file %s success!' % file_name)
else:
class Genbank:
def __init__(self, path):
"""
GenBank
"""
self.path = os.path.abspath(path)
self.info_dir = os.path.join(self.path, ".info")
self.assembly_summary_path = os.path.join(self.info_dir,
"assembly_summary.txt")
self.log = Logger("GenBank")
if not os.path.isdir(self.path):
os.mkdir(self.path)
if not os.path.isdir(self.info_dir):
os.mkdir(self.info_dir)
try:
self.assembly_summary = pd.read_csv(self.assembly_summary_path,
sep="\t",
index_col=0)
except FileNotFoundError:
self.assembly_summary = pd.read_csv(assembly_summary_url,
sep="\t",
index_col=0,
skiprows=1)
self.assembly_summary.to_csv(self.assembly_summary_path, sep="\t")
self.log.info("Downloaded assembly_summary.txt")
@property
def species(self):
for d in os.listdir(self.path):
species_path = os.path.join(self.path, d)
if not os.path.isdir(species_path):
continue
fastas = len(
[f for f in os.listdir(species_path) if f.endswith('fasta')])
if fastas < 10:
self.log.info("Not enough genomes for {}".format(d))
continue
yield Species.Species(species_path,
assembly_summary=self.assembly_summary)
def qc(self):
for species in self.species:
species.qc()
def update_data(log: Logger, data: Dict, wpaths: Dict) -> Dict:
"""Update data to link final files.
"""
data['silent_files']['assembly'] = list(
wpaths['outdir'].glob('*_hyb.silent'))
data['silent_files']['design'] = list(
wpaths['outdir'].glob('*_des.silent'))
data['minisilent']['assembly'] = wpaths['main'].joinpath(
'output_hyb.minisilent.gz')
data['minisilent']['design'] = wpaths['main'].joinpath(
'output_des.minisilent.gz')
for k in data['minisilent']:
log.info(f'Generating minisilent file at {data["minisilent"][k]}\n')
fd = gzip.open(data['minisilent'][k], "wb")
for line, _, _, _ in open_rosetta_file(
[str(x) for x in data['silent_files'][k]],
True,
check_symmetry=False):
fd.write(line.encode('utf-8'))
return data
class UrlToMarkdown(object):
default_mobilizer = "original"
def __init__(self, mobilizer="original"):
if mobilizer:
if not getattr(mobilizers, mobilizer.capitalize() + "Mobilizer", None):
raise Exception("Invalid mobilizer: {}".format(mobilizer))
self.default_mobilizer = mobilizer
self.log = Logger(self.__class__.__name__)
def convert(self, url, mobilizer=None, simple_result=True):
"""Fetch a page from URL, mobilize it, then convert it to Markdown
url: ...
mobilizer: 'original', 'instapaper',
simple_result: True returns markdown text, else returns a dict
"""
if not mobilizer:
mobilizer = self.default_mobilizer
try:
mob_object = getattr(mobilizers, mobilizer.capitalize() + "Mobilizer")
except AttributeError:
raise Exception("Invalid mobilizer: {}".format(mobilizer))
mob = mob_object()
self.log.debug(
"Obtaining {url} via {mobilizer}".format(url=url, mobilizer=mobilizer)
)
mobilized = mob.fetch(url)
self.log.info("Title is {0[title]!r}".format(mobilized))
self.log.debug("Converting {url} to Markdown".format(url=url))
h2t = HTML2Text()
# html2text also wraps image/link URLs, breaking them
h2t.body_width = 0
self.log.info("Converted to Markdown")
mobilized["markdown"] = h2t.handle(mobilized["body"].html())
if simple_result:
return mobilized["markdown"]
return mobilized
def main(thread_number, username, main_directory):
# set up logging
log_handler = FileHandler(join(main_directory, "log"))
log_handler.push_application()
log = Logger("fdsn_log")
def download_kernel(download_url):
log_message = f"[thread: {multiprocessing.current_process()}] start to download {download_url} "
log.info(log_message)
sh.wget("-c", download_url)
log_message = f"[thread: {multiprocessing.current_process()}] finish downloading {download_url} "
log.info(log_message)
filelist_ftp = get_ftp_file_list(username)
files_to_download = get_files_to_download(main_directory, filelist_ftp)
downloading_urls = get_download_urls(files_to_download)
with multiprocessing.Pool(thread_number) as pool:
pool.starmap(download_kernel, downloading_urls)
log.info("success")
class ConcoctR(object):
""" A class for running concoct from within python. Uses the
concoctParams class for representing input parameters. """
def __init__(self):
self.log = Logger('ConcoctR')
def run_concoct(self,
concoct_params,
drmaa_s=None,
drmaa_jt=None,
sbatch_script=None):
""" drmaa_s is a drama session
drmaa_jt is a drmaa job template """
if drmaa_s:
drmaa_jt.remoteCommand = "concoct " + " ".join(
concoct_params.args())
job_id = drmaa_s.runJob(drmaa_jt)
self.log.info("Jobid {0} with command: {1}".format(
drmaa_jt.remoteCommand, job_id))
return job_id
else:
if sbatch_script:
subprocess.Popen(['sbatch', sbatch_script])
else:
cla = ['concoct'] + concoct_params.args()
subprocess.Popen(cla)
self.log.info(
"Command line call with command: {0}".format(cla))
return None
def generate_sbatch_script(self, concoct_params, sbatch_params, file_name):
""" Generate a shell script that can be submitted with sbatch. """
with open(file_name, 'w+') as f:
f.write("#!/bin/bash" + '\n')
for sp in sbatch_params:
f.write("#SBATCH " + sp + '\n')
command = " ".join(['concoct'] + concoct_params.args())
f.write(command + '\n')
class Router(Thread):
"""Thread waiting for a request by another Driver and responding to
it with the chunked asked.
"""
def __init__(self, name, redis, get_chunk):
super(Router, self).__init__()
self.name = name
self.redis = redis
self.get_chunk = get_chunk
self.router = None
self.logger = Logger("{} - Router".format(self.name))
self.logger.info("Started")
self.context = zmq.Context.instance()
def run(self):
self.router = self.context.socket(zmq.ROUTER)
port = self.router.bind_to_random_port('tcp://*')
self.redis.set('drivers:{}:router'.format(self.name), port)
while True:
msg = self.router.recv_multipart()
self._respond_to(*msg)
def _respond_to(self, identity, filename, offset, size):
"""Calls the `get_chunk` handler defined by the Driver to get
the chunk and send it to the addressee.
"""
self.logger.debug(
"Getting chunk of size {} from offset {} in '{}'",
size, offset, filename
)
chunk = self.get_chunk(filename, int(offset), int(size))
self.router.send_multipart((identity, chunk))
class MyEventHandler(FileSystemEventHandler):
"""Logs all the events captured."""
def __init__(self, logfile, run_handler):
"""pass logfile to be opened and handler to flush writing the file"""
super(MyEventHandler, self).__init__()
self.run_log = Logger('Runs')
self.fs_log = Logger('Files')
socket_path = environ.get("NVIM_LISTEN_ADDRESS")
self.nvim = attach('socket', path=socket_path)
self.log_file = logfile
self.run_handler = run_handler
def on_moved(self, event):
"""called when a file or folder is moved"""
super(MyEventHandler, self).on_moved(event)
what = 'directory' if event.is_directory else 'file'
log_msg = "Moved {}: from {} to {}".format(what, event.src_path, event.dest_path)
self.fs_log.info(log_msg)
def on_created(self, event):
"""called on creation of a file or folder"""
super(MyEventHandler, self).on_created(event)
what = 'directory' if event.is_directory else 'file'
self.fs_log.info("Created {}: {}".format(what, event.src_path))
def on_deleted(self, event):
"""called on deletion of a file or folder"""
super(MyEventHandler, self).on_deleted(event)
what = 'directory' if event.is_directory else 'file'
self.fs_log.info("Deleted {}: {}".format(what, event.src_path))
def on_modified(self, event):
"""when a file is modified the event is logged and appended to a separate file
Then the script is run through python and the output is (over)written to self.log_file
and appended to the file handled by the info handler"""
super(MyEventHandler, self).on_modified(event)
what = 'directory' if event.is_directory else 'file'
self.fs_log.info("Modified {}: {}".format(what, event.src_path))
event_path = Path(event.src_path) # using plain src_path gives nonexistent path
if event_path.is_file():
in_file = str(event_path)
out_str = python(in_file)
self.run_log.notice('Output:\n{}'.format(out_str))
self.run_handler.close()
self.nvim.command('pedit ' + self.log_file)
class Logging_MixIn(Abstract_Interface):
"""
Logging Version
"""
def __init__(self, *args, **kwargs):
"""
Instantiate logging interface
:param args:
:param kwargs:
:return:
"""
Abstract_Interface.__init__(self, *args, **kwargs)
from logbook import Logger
self.log = Logger("{}-{}".format(type(self).__name__,self.door_name))
self.log.debug("Initialised using : {}".format(kwargs))
def activate(self):
self.log.debug("Activated")
def is_active(self):
self.log.debug("Checking activation")
def open(self, duration=10):
self.log.info("Opening for {}".format(duration))
class MediaWatcher(object):
default_config = {
'mock': False
}
def __init__(self, config):
self.config = MediaWatcherConfig(**config)
StreamHandler(sys.stdout).push_application()
self.logger = Logger('MediaWatcher')
self.mover = MediaWatcherMover(self.config, self.logger)
self.handler = MediaWatcherHandler(self.mover, self.logger)
self.observer = Observer()
self.setup_watch()
def startup(self):
self.logger.info('Performing initial scan.')
self.mover.scan()
self.logger.info('Initial scan done, ready to watch {} folder{}.'.format(len(self.config.watch), 's' if len(
self.config.watch) != 1 else ''))
def setup_watch(self):
for directory in self.config.watch:
path = Path(directory)
if not path.is_dir():
raise FileNotFoundError('Directory {} not found on system.'.format(directory))
self.observer.schedule(self.handler, str(path), recursive=True)
def watch(self):
self.observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
self.observer.stop()
self.observer.join()
class EventHandler(FileSystemEventHandler):#文件变化的触发
def __init__(self):
super(EventHandler,self).__init__()
self.log = Logger(u"fileMonitor")
def on_created(self, event):
super(EventHandler,self).on_created(event)
if os.path.splitext(event.src_path)[1] == ".py":
self.log.info("Create file: %s " % event.src_path)
# send plugin add message
pub.sendMessage(topics.PLUGIN_ADD, path=event.src_path)
def on_modified(self, event):
super(EventHandler,self).on_modified(event)
if os.path.splitext(event.src_path)[1] == ".py":
self.log.info("Modify file: %s " % event.src_path)
pub.sendMessage(topics.PLUGIN_MODIFY, path=event.src_path)
def on_deleted(self, event):
super(EventHandler,self).on_deleted(event)
if os.path.splitext(event.src_path)[1] == ".py":
self.log.info("delete file: %s " % event.src_path)
pub.sendMessage(topics.PLUGIN_DEL, path=event.src_path)
def main():
logger.info('Starting')
servers = [Server(i) for i in range(1, NUM_SERVERS + 1)]
start_servers(servers)
time.sleep(10)
try:
return test(servers)
except Exception, e:
logger.exception('Test failed: %s' % e)
return 1
finally:
logger.info('Stopping')
stop(servers)
if __name__ == '__main__':
format = '[{record.time}] {record.level_name:>5} [{record.extra[worker_id]}] {record.message}'
logging_setup = NestedSetup([
NullHandler(),
FileHandler(
filename=os.path.join(os.path.dirname(__file__), 'log/client.log'),
format_string=format,
bubble=True,
),
StderrHandler(level=logbook.INFO, format_string=format, bubble=True),
])
log = Logger('Main')
if len(sys.argv) > 1:
start = datetime.now()
letters = sys.argv[1]
preferred = None
if len(sys.argv) > 2:
preferred = list(sys.argv[2])
if letters:
letters = [x for x in letters]
log.info('Working with {}'.format(letters))
words = set()
with open('enable.txt') as dictionary:
words = set(word.strip().lower() for word in dictionary if len(word) <= len(letters) and len(word) > 1)
log.info('Evaluation against {} words'.format(len(words)))
log.info('Sorting words')
words = sorted(words, key=lambda word: len(word))
#max_length = max(words, key= lambda word: len(word))
#log.info('Max word length is {}'.format(max_length))
log.info('Creating worker pool')
#fix
#import dateutil
import arrow
import redis
#fix
import greenlet
import gevent
from gevent import Timeout
from gevent.pool import Pool
from logbook import Logger
log = Logger('heartbeat')
log.info('start')
#redis pool
REDIS_MAX_CONNECTIONS = 100
rpool = redis.ConnectionPool(host='localhost', port=6379, db=1, \
max_connections=REDIS_MAX_CONNECTIONS)
rclient = redis.Redis(connection_pool=rpool)
class TimeoutException(Exception):
""" timeout exception for gevent Timeout"""
pass
from logbook import Logger
from logbook import FileHandler
# max file size in lines (10M by default)
MAX_LINES_TO_READ = 10000000
# max file size in megabytes
MAX_FILE_SIZE = "500M"
# where to write application log
LOG_FILE_NAME = "reican.log"
TIMESTAMP_FORMAT = "YYYY-MM-DD HH:mm:ss"
log_handler = FileHandler(LOG_FILE_NAME)
log_handler.push_application()
log = Logger("Reican")
log.info("Logging started")
def die(msg=None):
"""
Print a message and exit
"""
if msg:
print
print msg
print
log.critical(msg)
sys.exit(1)
def usage():
from logbook import FileHandler
from logbook import Logger
from argparse import ArgumentParser
import sys
parser = ArgumentParser()
logpath = './log/'
parser.add_argument('--log', nargs=1, help='log path')
parser.add_argument('--version', nargs=1, help='maintain version')
args = parser.parse_args(sys.argv[1:])
logfilepath = logpath + args.log[0]
maintain_version = args.version[0]
log_handler = FileHandler(logfilepath)
logbk = Logger('Token Maintain')
with log_handler.applicationbound():
logbk.info('maintain prepare')
at_least = AT_LEAST_TOKEN_COUNT
max_tokens_redis_limit = MAX_TOKENS_IN_REDIS
logbk.info('maintain begin')
# 认证新用户，并将access_token加入mongodb，redis从mongodb导入新token，不重置已有token 的 req_count
if maintain_version == 'addatoken':
print 'generate new token, write to mongo, push to redis without reset request count'
generate_api_access_token(logbk)
add_without_reset_req_count(max_tokens_redis_limit, logbk)
# 将mongodb中所有access_token加入redis，并重置已有token 的 req_count
if maintain_version == 'addalltoken':
print 'push all tokens from mongo to redis and reset request count'
with error_handler.applicationbound():
Config = ConfigParser.ConfigParser()
Config.read('/etc/videodoor/videodoor.ini')
SensorPin = Config.getint('hardware','SensorPin')
log.debug('setting up GPIO pin %i...' % SensorPin)
GPIO.setmode(GPIO.BCM)
GPIO.setup(SensorPin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
log.debug('done.')
log.debug('Setting up the omxplayer instance...')
File = Config.get('video','File')
Options = Config.get('video','Options')
omx_status = False
log.info('initializing videoplayer with file %s and options %s' % (File, Options))
omx = OMXPlayer(File, Options)
log.debug('done.')
def start_video():
global omx, omx_status
if(omx_status):
log.warn('video already running')
else:
omx.toggle_pause()
omx_status = True
log.info('door opened')
def stop_video():
global omx, omx_status
if(omx_status):
class Admin(object):
"""Administrative task object."""
def __init__(self, conf):
"""conf: dict, yaml parameters."""
self.conf = conf
handler = TimedRotatingFileHandler(conf.log_file, date_format="%Y-%m-%d")
handler.push_application()
self.logger = Logger("Firetower-admin")
self.queue = redis_util.get_redis_conn(host=conf.redis_host, port=conf.redis_port, redis_db=conf.redis_db)
self.classifier = classifier.Levenshtein()
self.last_archive_run = None
# Wrote this to avoid numpy dependencies until we absolutely require them.
def mean_stdev(self, items):
"""Return mean and stdev of numerical list of items.
Args:
items: list, list of numbers (int or float) to perform calculations upon.
Returns:
tuble of (mean between items, and stdeviation of items and mean).
"""
n, mean, std = 0, 0, 0
if items:
n = len(items)
else:
return (0.0, 0.0)
for item in items:
mean = mean + item
mean = mean / float(n)
for item in items:
std = std + (item + mean) ** 2
std = sqrt(std / float(n)) # Avoid DivideByZero by not subtracting one.
return mean, std
def calc_stats(self):
"""Calculate and save mean and standard deviation."""
categories = category.Category.get_all_categories(self.queue)
for cat in categories:
all_events = cat.events.range(0, -1)
ratios = []
for event in all_events:
event = json.loads(event)
ratios.append(self.classifier.str_ratio(cat.signature, event["sig"]))
cat.mean, cat.stdev = self.mean_stdev(ratios)
def archive_events(self):
"""Run the timeseries archiving for all categories.
This code moves counts from the atomically incrementable HASHes
to Sorted Sets (which can be sliced by date)."""
now = datetime.datetime.utcnow()
if self.last_archive_run is None:
self.last_archive_run = datetime.datetime.utcnow()
return
delta = datetime.timedelta(seconds=self.conf.archive_time)
if self.last_archive_run < (now - delta):
self.logger.debug("Archiving counts older than %s seconds" % (self.conf.archive_time,))
for c in category.Category.get_all_categories(self.queue):
self.logger.debug("Archiving for %s category" % (c.cat_id))
c.timeseries.archive_cat_counts(self.last_archive_run)
def run(self, args):
"""Run set of jobs specified on commandline or config."""
self.logger.info("Running with tasks: %s" % (",".join(args)))
for arg in args:
if arg not in TASKS:
self.logger.error("Specified unknown task: %s" % (arg,))
sys.exit(1)
if arg == "calc_stats":
self.logger.info("Calculating stats for each category")
self.calc_stats()
if arg == "archive_events":
self.archive_events()
self.logger.info("Archiving old data from each category")
class Fibratus():
"""Fibratus entrypoint.
Setup the core components including the kernel
event stream collector and the tracing controller.
At this point the system handles are also being
enumerated.
"""
def __init__(self, filament):
self.logger = Logger(Fibratus.__name__)
self.file_handler = FileHandler(os.path.join(os.path.abspath(__file__), '..', '..', '..', 'fibratus.log'),
mode='w+')
self.kevt_streamc = KEventStreamCollector(etw.KERNEL_LOGGER_NAME.encode())
self.kcontroller = KTraceController()
self.ktrace_props = KTraceProps()
self.ktrace_props.enable_kflags()
self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME
self.handle_repository = HandleRepository()
self._handles = []
# query for handles on the
# start of kernel trace
with self.file_handler.applicationbound():
self.logger.info('Starting fibratus...')
self.logger.info('Enumerating system handles...')
self._handles = self.handle_repository.query_handles()
self.logger.info('%s handles found' % len(self._handles))
self.handle_repository.free_buffers()
self.thread_registry = ThreadRegistry(self.handle_repository, self._handles)
self.kevent = KEvent(self.thread_registry)
self._filament = filament
self.fsio = FsIO(self.kevent, self._handles)
self.hive_parser = HiveParser(self.kevent, self.thread_registry)
self.tcpip_parser = TcpIpParser(self.kevent)
self.dll_repository = DllRepository(self.kevent)
self.requires_render = {}
self.filters_count = 0
def run(self):
@atexit.register
def _exit():
self.stop_ktrace()
self.kcontroller.start_ktrace(etw.KERNEL_LOGGER_NAME, self.ktrace_props)
def on_kstream_open():
if self._filament is None:
IO.write_console('Done! ')
self.kevt_streamc.set_kstream_open_callback(on_kstream_open)
self._open_kstream()
def _open_kstream(self):
try:
self.kevt_streamc.open_kstream(self._on_next_kevent)
except Exception as e:
with self.file_handler.applicationbound():
self.logger.error(e)
except KeyboardInterrupt:
self.stop_ktrace()
def stop_ktrace(self):
IO.write_console('Stopping fibratus...')
if self._filament:
self._filament.close()
self.kcontroller.stop_ktrace(self.ktrace_props)
self.kevt_streamc.close_kstream()
def add_filters(self, kevent_filters):
if len(kevent_filters) > 0:
self.filters_count = len(kevent_filters)
# include the basic filters
# that are essential to the
# rest of kernel events
self.kevt_streamc.add_kevent_filter(ENUM_PROCESS)
self.kevt_streamc.add_kevent_filter(ENUM_THREAD)
self.kevt_streamc.add_kevent_filter(ENUM_IMAGE)
self.kevt_streamc.add_kevent_filter(REG_CREATE_KCB)
self.kevt_streamc.add_kevent_filter(REG_DELETE_KCB)
# these kevents are necessary for consistent state
# of the trace. If the user doesn't include them
# in a filter list, then we do the job but set the
# kernel event type as not eligible for rendering
if not KEvents.CREATE_PROCESS in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_PROCESS)
self.requires_render[CREATE_PROCESS] = False
else:
self.requires_render[CREATE_PROCESS] = True
if not KEvents.CREATE_THREAD in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_THREAD)
self.requires_render[CREATE_THREAD] = False
else:
self.requires_render[CREATE_THREAD] = True
if not KEvents.CREATE_FILE in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_FILE)
self.requires_render[CREATE_FILE] = False
else:
self.requires_render[CREATE_FILE] = True
for kevent_filter in kevent_filters:
ktuple = kname_to_tuple(kevent_filter)
if isinstance(ktuple, list):
for kt in ktuple:
self.kevt_streamc.add_kevent_filter(kt)
if not kt in self.requires_render:
self.requires_render[kt] = True
else:
self.kevt_streamc.add_kevent_filter(ktuple)
if not ktuple in self.requires_render:
self.requires_render[ktuple] = True
def _on_next_kevent(self, ktype, cpuid, ts, kparams):
"""Callback which fires when new kernel event arrives.
This callback is invoked for every new kernel event
forwarded from the kernel stream collector.
Parameters
----------
ktype: tuple
Kernel event type.
cpuid: int
Indentifies the CPU core where the event
has been captured.
ts: str
Temporal reference of the kernel event.
kparams: dict
Kernel event's parameters.
"""
# initialize kernel event properties
self.kevent.ts = ts
self.kevent.cpuid = cpuid
self.kevent.name = ktuple_to_name(ktype)
kparams = ddict(kparams)
# thread / process kernel events
if ktype in [CREATE_PROCESS,
CREATE_THREAD,
ENUM_PROCESS,
ENUM_THREAD]:
self.thread_registry.add_thread(ktype, kparams)
if ktype in [CREATE_PROCESS, CREATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent,
ktype,
kparams)
self._render(ktype)
elif ktype in [TERMINATE_PROCESS, TERMINATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent,
ktype,
kparams)
self._render(ktype)
self.thread_registry.remove_thread(ktype, kparams)
# file system/disk kernel events
elif ktype in [CREATE_FILE,
DELETE_FILE,
CLOSE_FILE,
READ_FILE,
WRITE_FILE]:
self.fsio.parse_fsio(ktype, kparams)
self._render(ktype)
# dll kernel events
elif ktype in [LOAD_IMAGE, ENUM_IMAGE]:
self.dll_repository.register_dll(kparams)
if ktype == LOAD_IMAGE:
self._render(ktype)
elif ktype == UNLOAD_IMAGE:
self.dll_repository.unregister_dll(kparams)
self._render(ktype)
# registry kernel events
elif ktype == REG_CREATE_KCB:
self.hive_parser.add_kcb(kparams)
elif ktype == REG_DELETE_KCB:
self.hive_parser.remove_kcb(kparams.key_handle)
elif ktype in [REG_CREATE_KEY,
REG_DELETE_KEY,
REG_OPEN_KEY,
REG_QUERY_KEY,
REG_SET_VALUE,
REG_DELETE_VALUE,
REG_QUERY_VALUE]:
self.hive_parser.parse_hive(ktype, kparams)
self._render(ktype)
# network kernel events
elif ktype in [SEND_SOCKET_TCPV4,
SEND_SOCKET_UDPV4,
RECV_SOCKET_TCPV4,
RECV_SOCKET_UDPV4,
ACCEPT_SOCKET_TCPV4,
CONNECT_SOCKET_TCPV4,
DISCONNECT_SOCKET_TCPV4,
RECONNECT_SOCKET_TCPV4]:
self.tcpip_parser.parse_tcpip(ktype, kparams)
self._render(ktype)
if self._filament:
# call filament method
# to process the next
# kernel event from the stream
if ktype not in [ENUM_PROCESS,
ENUM_THREAD, ENUM_IMAGE]:
if self.kevent.name:
self._filament.process(self.kevent)
def _render(self, ktype):
"""Renders the kevent to the standard output stream.
Parameters
----------
ktype: tuple
Identifier of the kernel event
"""
if not self._filament:
if ktype in self.requires_render:
rr = self.requires_render[ktype]
if rr:
self.kevent.render()
elif self.filters_count == 0:
self.kevent.render()
class Worker(object):
redis_worker_namespace_prefix = 'rq:worker:'
redis_workers_keys = 'rq:workers'
@classmethod
def all(cls, connection=None):
"""Returns an iterable of all Workers.
"""
if connection is None:
connection = get_current_connection()
reported_working = connection.smembers(cls.redis_workers_keys)
workers = [cls.find_by_key(key, connection) for key in
reported_working]
return compact(workers)
@classmethod
def find_by_key(cls, worker_key, connection=None):
"""Returns a Worker instance, based on the naming conventions for
naming the internal Redis keys. Can be used to reverse-lookup Workers
by their Redis keys.
"""
prefix = cls.redis_worker_namespace_prefix
name = worker_key[len(prefix):]
if not worker_key.startswith(prefix):
raise ValueError('Not a valid RQ worker key: %s' % (worker_key,))
if connection is None:
connection = get_current_connection()
if not connection.exists(worker_key):
return None
name = worker_key[len(prefix):]
worker = cls([], name, connection=connection)
queues = connection.hget(worker.key, 'queues')
worker._state = connection.hget(worker.key, 'state') or '?'
if queues:
worker.queues = [Queue(queue, connection=connection)
for queue in queues.split(',')]
return worker
def __init__(self, queues, name=None, default_result_ttl=DEFAULT_RESULT_TTL,
connection=None, exc_handler=None): # noqa
if connection is None:
connection = get_current_connection()
self.connection = connection
if isinstance(queues, Queue):
queues = [queues]
self._name = name
self.queues = queues
self.validate_queues()
self._exc_handlers = []
self.default_result_ttl = default_result_ttl
self._state = 'starting'
self._is_horse = False
self._horse_pid = 0
self._stopped = False
self.log = Logger('worker')
self.failed_queue = get_failed_queue(connection=self.connection)
# By default, push the "move-to-failed-queue" exception handler onto
# the stack
self.push_exc_handler(self.move_to_failed_queue)
if exc_handler is not None:
self.push_exc_handler(exc_handler)
def validate_queues(self): # noqa
"""Sanity check for the given queues."""
if not iterable(self.queues):
raise ValueError('Argument queues not iterable.')
for queue in self.queues:
if not isinstance(queue, Queue):
raise NoQueueError('Give each worker at least one Queue.')
def queue_names(self):
"""Returns the queue names of this worker's queues."""
return map(lambda q: q.name, self.queues)
def queue_keys(self):
"""Returns the Redis keys representing this worker's queues."""
return map(lambda q: q.key, self.queues)
@property # noqa
def name(self):
"""Returns the name of the worker, under which it is registered to the
monitoring system.
By default, the name of the worker is constructed from the current
(short) host name and the current PID.
"""
if self._name is None:
hostname = socket.gethostname()
shortname, _, _ = hostname.partition('.')
self._name = '%s.%s' % (shortname, self.pid)
return self._name
@property
def key(self):
"""Returns the worker's Redis hash key."""
return self.redis_worker_namespace_prefix + self.name
@property
def pid(self):
"""The current process ID."""
return os.getpid()
@property
def horse_pid(self):
"""The horse's process ID. Only available in the worker. Will return
0 in the horse part of the fork.
"""
return self._horse_pid
@property
def is_horse(self):
"""Returns whether or not this is the worker or the work horse."""
return self._is_horse
def procline(self, message):
"""Changes the current procname for the process.
This can be used to make `ps -ef` output more readable.
"""
setprocname('rq: %s' % (message,))
def register_birth(self): # noqa
"""Registers its own birth."""
self.log.debug('Registering birth of worker %s' % (self.name,))
if self.connection.exists(self.key) and \
not self.connection.hexists(self.key, 'death'):
raise ValueError(
'There exists an active worker named \'%s\' '
'already.' % (self.name,))
key = self.key
now = time.time()
queues = ','.join(self.queue_names())
with self.connection.pipeline() as p:
p.delete(key)
p.hset(key, 'birth', now)
p.hset(key, 'queues', queues)
p.sadd(self.redis_workers_keys, key)
p.execute()
def register_death(self):
"""Registers its own death."""
self.log.debug('Registering death')
with self.connection.pipeline() as p:
# We cannot use self.state = 'dead' here, because that would
# rollback the pipeline
p.srem(self.redis_workers_keys, self.key)
p.hset(self.key, 'death', time.time())
p.expire(self.key, 60)
p.execute()
def set_state(self, new_state):
self._state = new_state
self.connection.hset(self.key, 'state', new_state)
def get_state(self):
return self._state
state = property(get_state, set_state)
@property
def stopped(self):
return self._stopped
def _install_signal_handlers(self):
"""Installs signal handlers for handling SIGINT and SIGTERM
gracefully.
"""
def request_force_stop(signum, frame):
"""Terminates the application (cold shutdown).
"""
self.log.warning('Cold shut down.')
# Take down the horse with the worker
if self.horse_pid:
msg = 'Taking down horse %d with me.' % self.horse_pid
self.log.debug(msg)
try:
os.kill(self.horse_pid, signal.SIGKILL)
except OSError as e:
# ESRCH ("No such process") is fine with us
if e.errno != errno.ESRCH:
self.log.debug('Horse already down.')
raise
raise SystemExit()
def request_stop(signum, frame):
"""Stops the current worker loop but waits for child processes to
end gracefully (warm shutdown).
"""
self.log.debug('Got signal %s.' % signal_name(signum))
signal.signal(signal.SIGINT, request_force_stop)
signal.signal(signal.SIGTERM, request_force_stop)
msg = 'Warm shut down requested.'
self.log.warning(msg)
# If shutdown is requested in the middle of a job, wait until
# finish before shutting down
if self.state == 'busy':
self._stopped = True
self.log.debug('Stopping after current horse is finished. '
'Press Ctrl+C again for a cold shutdown.')
else:
raise StopRequested()
signal.signal(signal.SIGINT, request_stop)
signal.signal(signal.SIGTERM, request_stop)
def work(self, burst=False): # noqa
"""Starts the work loop.
Pops and performs all jobs on the current list of queues. When all
queues are empty, block and wait for new jobs to arrive on any of the
queues, unless `burst` mode is enabled.
The return value indicates whether any jobs were processed.
"""
self._install_signal_handlers()
did_perform_work = False
self.register_birth()
self.log.info('RQ worker started, version %s' % VERSION)
self.state = 'starting'
try:
while True:
if self.stopped:
self.log.info('Stopping on request.')
break
self.state = 'idle'
qnames = self.queue_names()
self.procline('Listening on %s' % ','.join(qnames))
self.log.info('')
self.log.info('*** Listening on %s...' % \
green(', '.join(qnames)))
wait_for_job = not burst
try:
result = Queue.dequeue_any(self.queues, wait_for_job, \
connection=self.connection)
if result is None:
break
except StopRequested:
break
except UnpickleError as e:
msg = '*** Ignoring unpickleable data on %s.' % \
green(e.queue.name)
self.log.warning(msg)
self.log.debug('Data follows:')
self.log.debug(e.raw_data)
self.log.debug('End of unreadable data.')
self.failed_queue.push_job_id(e.job_id)
continue
self.state = 'busy'
job, queue = result
# Use the public setter here, to immediately update Redis
job.status = Status.STARTED
self.log.info('%s: %s (%s)' % (green(queue.name),
blue(job.description), job.id))
self.fork_and_perform_job(job)
did_perform_work = True
finally:
if not self.is_horse:
self.register_death()
return did_perform_work
def fork_and_perform_job(self, job):
"""Spawns a work horse to perform the actual work and passes it a job.
The worker will wait for the work horse and make sure it executes
within the given timeout bounds, or will end the work horse with
SIGALRM.
"""
child_pid = os.fork()
if child_pid == 0:
self.main_work_horse(job)
else:
self._horse_pid = child_pid
self.procline('Forked %d at %d' % (child_pid, time.time()))
while True:
try:
os.waitpid(child_pid, 0)
break
except OSError as e:
# In case we encountered an OSError due to EINTR (which is
# caused by a SIGINT or SIGTERM signal during
# os.waitpid()), we simply ignore it and enter the next
# iteration of the loop, waiting for the child to end. In
# any other case, this is some other unexpected OS error,
# which we don't want to catch, so we re-raise those ones.
if e.errno != errno.EINTR:
raise
def main_work_horse(self, job):
"""This is the entry point of the newly spawned work horse."""
# After fork()'ing, always assure we are generating random sequences
# that are different from the worker.
random.seed()
# Always ignore Ctrl+C in the work horse, as it might abort the
# currently running job.
# The main worker catches the Ctrl+C and requests graceful shutdown
# after the current work is done. When cold shutdown is requested, it
# kills the current job anyway.
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self._is_horse = True
self.log = Logger('horse')
success = self.perform_job(job)
# os._exit() is the way to exit from childs after a fork(), in
# constrast to the regular sys.exit()
os._exit(int(not success))
def perform_job(self, job):
"""Performs the actual work of a job. Will/should only be called
inside the work horse's process.
"""
self.procline('Processing %s from %s since %s' % (
job.func_name,
job.origin, time.time()))
try:
with death_penalty_after(job.timeout or 180):
rv = job.perform()
# Pickle the result in the same try-except block since we need to
# use the same exc handling when pickling fails
pickled_rv = dumps(rv)
job._status = Status.FINISHED
except:
# Use the public setter here, to immediately update Redis
job.status = Status.FAILED
self.handle_exception(job, *sys.exc_info())
return False
if rv is None:
self.log.info('Job OK')
else:
self.log.info('Job OK, result = %s' % (yellow(unicode(rv)),))
# How long we persist the job result depends on the value of
# result_ttl:
# - If result_ttl is 0, cleanup the job immediately.
# - If it's a positive number, set the job to expire in X seconds.
# - If result_ttl is negative, don't set an expiry to it (persist
# forever)
result_ttl = self.default_result_ttl if job.result_ttl is None else job.result_ttl # noqa
if result_ttl == 0:
job.delete()
self.log.info('Result discarded immediately.')
else:
p = self.connection.pipeline()
p.hset(job.key, 'result', pickled_rv)
p.hset(job.key, 'status', job._status)
if result_ttl > 0:
p.expire(job.key, result_ttl)
self.log.info('Result is kept for %d seconds.' % result_ttl)
else:
self.log.warning('Result will never expire, clean up result key manually.')
p.execute()
return True
def handle_exception(self, job, *exc_info):
"""Walks the exception handler stack to delegate exception handling."""
exc_string = ''.join(
traceback.format_exception_only(*exc_info[:2]) +
traceback.format_exception(*exc_info))
self.log.error(exc_string)
for handler in reversed(self._exc_handlers):
self.log.debug('Invoking exception handler %s' % (handler,))
fallthrough = handler(job, *exc_info)
# Only handlers with explicit return values should disable further
# exc handling, so interpret a None return value as True.
if fallthrough is None:
fallthrough = True
if not fallthrough:
break
def move_to_failed_queue(self, job, *exc_info):
"""Default exception handler: move the job to the failed queue."""
exc_string = ''.join(traceback.format_exception(*exc_info))
self.log.warning('Moving job to %s queue.' % self.failed_queue.name)
self.failed_queue.quarantine(job, exc_info=exc_string)
def push_exc_handler(self, handler_func):
"""Pushes an exception handler onto the exc handler stack."""
self._exc_handlers.append(handler_func)
def pop_exc_handler(self):
"""Pops the latest exception handler off of the exc handler stack."""
return self._exc_handlers.pop()
def post_created(response):
if response.status_code == 201:
return True
else:
print response.text
print response.status_code
return False
# Questionnaire
questionnaire = {'name': 'FMR Continuation Desire Questionnaire'}
r = requests.post('{}/questionnaire'.format(BASE_URL),
data=json.dumps(questionnaire),
headers={'content-type': 'application/json'})
if r.status_code == 201:
log.info('Questionnaire {} with status code {}'.format(questionnaire['name'], r.status_code))
else:
log.error('Questionnaire {} with status code {}'.format(questionnaire['name'], r.status_code))
# QuestionSet
question_sets = [
{'name': 'Before the experience', 'sort_nr': '1',
'info_text': 'Questions that need to be filled out before you start playing',
'questionnaire_id': '1'},
{'name': 'During the experience', 'sort_nr': '2',
'info_text': 'Questions regarding playing the game', 'questionnaire_id': '1'},
{'name': 'After the experience', 'sort_nr': '3',
'info_text': 'Questions regarding post-play', 'questionnaire_id': '1'}
]
def main(run_name, settings, exec_mode):
# File path and test data path
fp = os.path.dirname(__file__)
tdp = join(fp,"..", "tests", "test_data")
composition = settings.get("composition_file", join(tdp,"composition.fa"))
coverage = settings.get("coverage_file", join(tdp,"coverage"))
result_path = settings.get("results_path_base", join(fp,"..","tmp_out_test"))
kmer_lengths = settings.get("kmer_lengths", [4])
pcas = settings.get("total_percentage_pca", [80])
thresholds = settings.get("length_threshold", [1000])
cv_types = settings.get("covariance_type", ["full"])
clusters = settings.get("clusters", "2,100,2")
max_n_processors = settings.get("max_n_processors", 1)
email = settings.get("email", None)
log_path = settings.get("log_path",
join(os.path.expanduser("~"),"log","concoctr.log"))
handler = TimedRotatingFileHandler(log_path)
logger = Logger(run_name)
handler.push_application()
result_rows = []
indx = []
con_ps = []
if exec_mode == 'drmaa':
s = drmaa.Session()
s.initialize()
result_dir = os.path.join(result_path, run_name)
os.mkdir(result_dir)
slurm_dir = os.path.join(result_dir, 'slurm')
os.mkdir(slurm_dir)
sbatch_dir = os.path.join(result_dir, 'sbatch')
os.mkdir(sbatch_dir)
concoct_dir = os.path.join(result_dir, 'concoct_output')
os.mkdir(concoct_dir)
for k in kmer_lengths:
for pca in pcas:
for thr in thresholds:
for cv in cv_types:
job_name = "_".join(map(str, [k, pca, thr, cv]))
con_p = ConcoctParams(composition,
coverage,
kmer_length = k,
total_percentage_pca= pca,
length_threshold = thr,
covariance_type = cv,
basename = os.path.join(concoct_dir, job_name) + "/",
max_n_processors = max_n_processors,
clusters = clusters)
con_ps.append(con_p)
cr = ConcoctR()
if (k > 9):
# Throw in some extra memory
n_cores = 4
else:
n_cores = 1
if exec_mode == 'drmaa':
jt = s.createJobTemplate()
jt.nativeSpecification = '-A b2010008 -p core -n {} -t 7-00:00:00'.format(n_cores)
jt.email = email
jt.workingDirectory = result_path
jobid = cr.run_concoct(con_p, drmaa_s=s, drmaa_jt=jt)
elif exec_mode == 'sbatch':
script_file = os.path.join(result_dir, 'sbatch', job_name)
sbatch_params = ['-A b2010008',
'-p core',
'-n {}'.format(n_cores),
'-t 7-00:00:00',
"-J {}".format(job_name),
"-o {}".format(os.path.join(result_dir, 'slurm', 'slurm-%j.out'))]
cr.generate_sbatch_script(con_p, sbatch_params, script_file)
jobid = cr.run_concoct(con_p, sbatch_script = script_file)
if jobid:
result_rows.append(con_p.options)
indx.append(jobid)
logger.info("Submitted jobid {0}".format(jobid))
results_df = p.DataFrame(result_rows, index=indx)
results_df.to_csv(os.path.join(result_path, run_name + "_all_results.csv"))
handler.pop_application()
for pid, project_data in all_projects:
experiment_settings = exp_settings_class()
manager = manager_class(experiment_settings)
try:
project, settings = manager.get_project(pid)
try:
manager.recheck_folders_and_params(pid, project, project_data=project_data)
except OSError, e:
app_logger.error("OSError: %s" % e)
app_logger.warning("Project %s was added early" % pid)
except AttributeError, e:
app_logger.error("ERROR: please, check that all settings given as dictionaries, not sets. (%s)" % e)
raise Exception(e)
except ProjectManagerException, e:
manager.add_project(pid, project_data, init=True)
app_logger.info("Project %s added" % pid)
def execute(args, usage, dataset_dict, exp_settings, exp_class, manager_class):
''' Execute experiment with givent args.
'''
if not len(args) in [5,7]:
print usage
sys.exit(0)
command = args[0]
tasks = args[1]
start = int(args[2])
end = int(args[3])
dataset = args[4]
settings_context = None
project_context = None
class PortfolioManager:
'''
Observes the trader universe and produces
orders to be excuted within zipline
Abstract method meant to be used by user
to construct their portfolio optimizer
'''
__metaclass__ = abc.ABCMeta
#TODO Add in the constructor or setup parameters some general settings like maximum weights, positions, frequency,...
def __init__(self, parameters):
'''
Parameters
parameters : dict(...)
Named parameters used either for general portfolio settings
(server and constraints), and for user optimizer function
'''
super(PortfolioManager, self).__init__()
self.log = Logger('Manager')
self.portfolio = None
self.date = None
self._optimizer_parameters = parameters
self.connected = False
self.server = parameters.get('server', None)
#TODO Message emission only if a client exists ? Could try to bind and give up if no connections
#NOTE Non blocking recv(): https://github.com/zeromq/pyzmq/issues/132 / zmq.NOBLOCK ?
#NOTE good example: http://zguide2.zeromq.org/py:peering3
#if self.server.ports is not None:
#startup_msg = self.server.receive()
#self.connected = True
#log.info(json.dumps(startup_msg, indent=4, separators=(',', ': ')))
#TODO Should send stuff anyway, and accept new connections while running
#else:
self.connected = parameters.get('connected', False)
# Run the server if the engine didn't while it is asked
if self.server.port is None and self.connected:
self.log.info('Binding manager on default port...')
self.server.run(host='127.0.0.1', port=5570)
@abc.abstractmethod
def optimize(self):
'''
Users must overwrite this method
'''
pass
def update(self, portfolio, date):
'''
Actualizes the portfolio universe
and if connected, sends it through the wires
________________________________
Parameters
portfolio: zipline.portfolio(1)
ndict object storing portfolio values at the given date
date: datetime.datetime(1)
Current date in zipline simulation
'''
self.portfolio = portfolio
self.date = date
#FIXME A generic method: f(ndict) = dict()
portfolio.capital_used = portfolio.capital_used[0]
#portfolio.start_date = portfolio.start_date.strftime(format='%Y-%m-%d %H:%M')
#FIXME remote console receives nothing
if self.connected:
self.server.send({'positions': json.loads(str(portfolio.positions).replace('Position(', '').replace(')', '').replace("'", '"')),
'value': portfolio.portfolio_value,
'cash': portfolio.cash,
'returns': portfolio.returns,
'pnl': portfolio.pnl,
'capital_used': portfolio.capital_used,
'actif': portfolio.positions_value},
type='portfolio',
channel='dashboard')
def trade_signals_handler(self, signals):
'''
Process buy and sell signals from backtester or live trader
@param signals: dict holding stocks of interest, format like {"google": 567.89, "apple": -345.98}
If the value is negative -> sell signal, otherwize buy one
@return: dict orderBook, like {"google": 34, "apple": -56}
'''
orderBook = dict()
# If value < 0, it's a sell signal on the key, else buy signal
to_buy = [t for t in signals if signals[t] > 0]
to_sell = set(self.portfolio.positions.keys()).intersection([t for t in signals if signals[t] < 0])
if not to_buy and not to_sell:
# Nothing to do
return dict()
# Compute the optimal portfolio allocation, using user defined function
alloc, e_ret, e_risk = self.optimize(self.date, to_buy, to_sell, self._optimizer_parameters)
#TODO Check about selling in available money and handle 250 stocks limit
#TODO Handle max_* as well, ! already actif stocks
# Building orders for zipline
for t in alloc:
# Handle allocation returned as number of stocks to order
if isinstance(alloc[t], int):
orderBook[t] = alloc[t]
# Handle allocation returned as stock weights to order
elif isinstance(alloc[t], float):
# Sell orders
if alloc[t] <= 0:
orderBook[t] = int(alloc[t] * self.portfolio.positions[t].amount)
# Buy orders
else:
# If we already trade this ticker, substract owned amount before computing number of stock to buy
if self.portfolio.positions[t].amount:
price = self.portfolio.positions[t].last_sale_price
else:
price = signals[t]
orderBook[t] = (int(alloc[t] * self.portfolio.portfolio_value / price)
- self.portfolio.positions[t].amount)
return orderBook
def setup_strategie(self, parameters):
'''
General parameters or user ones setting
(maw_weigth, max_assets, max_frequency, commission cost)
________________________________________________________
Parameters
parameters: dict(...)
Arbitrary values to change general constraints,
or for user algorithm settings
'''
for name, value in parameters.iteritems():
self._optimizer_parameters[name] = value
