def main(in_folder, out_folder, pipeline, num_threads, log_file=None):
if not os.path.isdir(in_folder):
print("Input folder %s does not exist" % in_folder)
if not os.path.isdir(out_folder):
print("Creating output folder %s" % out_folder)
os.makedirs(out_folder)
set_logger(log_file)
p = pipeline_registry.all[pipeline](in_folder, out_folder, num_threads)
p.run()
def main(parser, appConfig):
import utils.logger as logger
(options, args) = parser.parse_args(args=None, values=None)
loggerConfig = util.DotDict(logging_config=options.log_config,
file=APPNAME + '.log',
name=options.logger)
logger_dictConfig = logger.set_logger(loggerConfig, options.log_path,
options.log_level)
validate_options_config(options)
util.print_options_attrs2(options, [
("springboot", "version"),
("group", "groupId"),
("artifact", "artifactId"),
("name", "name"),
("ver", "artifact.version"),
("description", "description"),
("packagename", "packageName"),
("packaging", "packaging"),
("java", "java.version"),
("port", "server.port"),
("context_path", "server.servlet.context-path"),
])
create_sb2web_project(appConfig, options)
pass
def main(argv=None):
print("start of main")
main_time = time.time()
os.makedirs(RESULT_DIR)
# loging
LOG_FILE = os.path.join(RESULT_DIR, "log.txt")
logger.set_logger(level=FLAGS.get('log_level'),
stream=True,
fileh=True,
filename=LOG_FILE)
# file handling
logger.info("create folder for results: {}".format(RESULT_DIR))
if FLAGS.checkpoint_step > 0:
os.mkdir(CHECKPOINT_DIR)
logger.info("create checkpoints folder: {}".format(CHECKPOINT_DIR))
# import the corresponding module
# what about models.model ????????
try:
model_path = 'models.' + FLAGS.get('model').lower()
model_module = __import__(model_path, fromlist=[''])
except ImportError:
raise ValueError("no such model exists: {}".format())
# parse all FLAGS
FLAGS.parse_and_log()
# start training
train(model_module.Model)
# pring something before end
logger.newline(2)
logger.info("total time used: {}".format(time.time() - main_time))
logger.info("summary dir: " + RESULT_DIR)
logger.newline()
logger.info("~end of main~")
def setup_callables(self):
monitor = "val_dice_coef"
# Setup callback to save best weights after each epoch
checkpointer = ModelCheckpoint(filepath=os.path.join(
self.model_dir, 'weights.{epoch:02d}-{val_loss:.2f}.hdf5'),
verbose=0,
save_best_only=True,
save_weights_only=True,
monitor=monitor,
mode='max')
# setup callback to register training history
csv_logger = CSVLogger(os.path.join(self.log_dir, 'log.csv'),
append=True,
separator=';')
# setup logger to catch warnings and info messages
set_logger(os.path.join(self.log_dir, 'train_val.log'))
# setup callback to retrieve tensorboard info
tensorboard = TensorBoard(log_dir=self.log_dir,
write_graph=True,
histogram_freq=0)
# setup early stopping to stop training if val_loss is not increasing after 3 epochs
early_stopping = EarlyStopping(monitor=monitor,
patience=5,
mode='max',
verbose=0)
lr_reducer = ReduceLROnPlateau(monitor=monitor,
factor=0.05,
cooldown=0,
patience=5,
verbose=0,
mode='max')
return [
checkpointer, csv_logger, tensorboard, early_stopping, lr_reducer
]
def __init__(self, command_prefix: str, intents: discord.Intents,
**kwargs):
super().__init__(command_prefix=command_prefix,
intents=intents,
**kwargs)
self.logger = set_logger()
self.verification_queue = dict()
self.event_queue = Queue()
self.obj_cache = Cache()
self.running = True
self.default_invite = \
"https://discord.com/api/oauth2/authorize?client_id=767842408758771742&permissions=51200&scope=bot"
self.reddit = self.create_reddit_connection()
self.load_data()
def main(parser, appConfig):
import utils.logger as logger
(options, args) = parser.parse_args(args=None, values=None)
loggerConfig = {
'logging_config': options.log_config,
'file': APPNAME + '.log',
'name': options.logger
}
logger_dictConfig = logger.set_logger(loggerConfig, options.log_path,
options.log_level)
# 设置模板环境
tmplProjectDir = options.input
templatesDir = os.path.join(tmplProjectDir, "templates")
j2env = Environment(loader=FileSystemLoader(templatesDir))
# 处理每个配置文件
num = 0
flist = os.listdir(tmplProjectDir)
for name in flist:
_, extname = os.path.splitext(name)
if extname == ".yaml":
configYaml = os.path.join(tmplProjectDir, name)
if not os.path.isdir(configYaml):
# 载入 yaml 配置文件
fd = open(configYaml)
dictcfg = yaml.load(fd.read())
fd.close()
num += 1
util.info("[%d] processing config: %s" % (num, configYaml))
generate(parser, dictcfg, templatesDir, j2env, options)
util.info("success: total %d config file(s) processed." % num)
pass
import os
import sys
import time
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
from utils.folder import make_folder
from utils.logger import set_logger
logger = set_logger("trend")
base_path = os.path.dirname(os.path.abspath('..'))
data_path = os.path.join(base_path, 'data')
save_path = os.path.join(data_path, 'stock')
folder_path = os.path.join(save_path, 'folder')
day_path = os.path.join(folder_path, '일봉_20190323')
invest_path = os.path.join(base_path, 'invest')
trend_path = os.path.join(invest_path, '3.추세주')
result_path = os.path.join(trend_path, 'result')
make_folder(invest_path, trend_path, result_path)
def load_day_data(files, code_list, standard_day):
df_list = []
for i, file in enumerate(files):
try:
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
from utils.folder import make_folder
from utils.logger import set_logger
base_path = os.path.dirname(os.path.abspath('..'))
data_path = os.path.join(base_path, 'data')
stock_path = os.path.join(data_path, 'stock')
folder_path = os.path.join(stock_path, 'folder')
minute_path = os.path.join(folder_path, '분별매매가격_가상화폐')
code_path = os.path.join(base_path, 'code')
crawler_path = os.path.join(code_path, 'crawler')
make_folder(base_path, data_path, stock_path, folder_path, minute_path, code_path, crawler_path)
logger = set_logger('minute_logger')
def main(code, dates):
for date in dates:
tables = []
for num in range(1, 43):
page = "https://finance.naver.com/item/sise_time.nhn?code=" + str(code) + "&thistime=2019" + str(date) + "160000&page=" + str(num)
html = urlopen(page)
bs_object = BeautifulSoup(html, "html.parser")
bs_object_table = bs_object.table
tables.append(bs_object_table)
sleep(0.1)
# data frame
GROUP_VMI = config_raw.get('VirtualMachineImage', 'group')
PLURAL_VM_DISK = config_raw.get('VirtualMachineDisk', 'plural')
VERSION_VM_DISK = config_raw.get('VirtualMachineDisk', 'version')
GROUP_VM_DISK = config_raw.get('VirtualMachineDisk', 'group')
PLURAL_VM_SNAPSHOT = config_raw.get('VirtualMachineSnapshot', 'plural')
VERSION_VM_SNAPSHOT = config_raw.get('VirtualMachineSnapshot', 'version')
GROUP_VM_SNAPSHOT = config_raw.get('VirtualMachineSnapshot', 'group')
PLURAL_BLOCK_DEV_UIT = config_raw.get('VirtualMahcineBlockDevUit', 'plural')
VERSION_BLOCK_DEV_UIT = config_raw.get('VirtualMahcineBlockDevUit', 'version')
GROUP_BLOCK_DEV_UIT = config_raw.get('VirtualMahcineBlockDevUit', 'group')
LABEL = 'host=%s' % (socket.gethostname())
TIMEOUT = config_raw.get('WatcherTimeout', 'timeout')
logger = logger.set_logger(os.path.basename(__file__), '/var/log/virtctl.log')
'''
Handle support CMDs settings in default.cfg.
NOTE: if the key ends up with 'WithNameField' means that the CMD is using 'name' variable as index.
The key ends up with 'WithDomainField' means that the CMD is using 'domain' variable as index.
The key ends up with 'WithVolField' means that the CMD is using 'vol' variable as index.
'''
ALL_SUPPORT_CMDS = {}
ALL_SUPPORT_CMDS_WITH_NAME_FIELD = {}
ALL_SUPPORT_CMDS_WITH_DOMAIN_FIELD = {}
ALL_SUPPORT_CMDS_WITH_VOL_FIELD = {}
ALL_SUPPORT_CMDS_WITH_SNAPNAME_FIELD = {}
for k,v in config_raw._sections.items():
if string.find(k, 'SupportCmds') != -1:
split = arg.find("=")
setattr(args, arg[2:split], evol(arg[split+1:]))
return args
if __name__ == "__main__":
args = parse_args()
if args.jobid is not None:
args.run_name = args.run_name + '-' + args.jobid
# if args.losswise_tag is not None:
# args.losswise_tag = args.losswise_tag + '-' + args.jobid
try:
args.shortname = args.run_name
except:
setattr(args, "shortname", args.run_name)
# create dir for saving
args.saverootpath = osp.abspath(args.saverootpath)
savepath = osp.join(args.saverootpath, args.run_name)
if not osp.exists(savepath):
os.makedirs(savepath)
# np.random.seed(args.seed)
# random.seed(args.seed)
logger = set_logger(name=args.shortname, level=args.loglevel,
filepath=osp.join(savepath, 'log.txt'))
logger.info("=> Training mode")
train(args)
import traceback
from xml.etree.ElementTree import fromstring
from xmljson import badgerfish as bf
from json import dumps, loads
from sys import exit
from utils.exception import *
# from utils.libvirt_util import get_pool_info, get_volume_xml, get_volume_path, get_volume_snapshots, is_pool_started, \
# is_pool_defined
from utils.libvirt_util import get_xml, vm_state
from utils.utils import *
from utils import logger
LOG = "/var/log/kubesdvm.log"
logger = logger.set_logger(os.path.basename(__file__), LOG)
class Executor(object):
def __init__(self, cmd, params, with_result=False):
if cmd is None or cmd == "":
raise Exception("plz give me right cmd.")
if not isinstance(params, dict):
raise Exception("plz give me right parameters.")
self.params = params
self.cmd = cmd
self.with_result = with_result
def get_cmd(self):
cmd = self.cmd
'''
Import third party libs
'''
from kubernetes import client, watch
from kubernetes.client import V1DeleteOptions
from libvirt import libvirtError
'''
Import local libs
'''
from utils.libvirt_util import destroy, \
create, is_vm_active, is_vm_exists
from utils import logger, constants
from utils.utils import ExecuteException, \
report_failure, randomUUID, now_to_datetime, get_hostname_in_lower_case, UserDefinedEvent
logger = logger.set_logger(os.path.basename(__file__),
constants.VIRTCTL_DEBUG_LOG)
class Executor():
def __init__(self, policy, invoke_cmd, query_cmd):
self.policy = policy
self.invoke_cmd = invoke_cmd
self.query_cmd = query_cmd
def execute(self):
if not self.invoke_cmd or not self.query_cmd:
logger.debug('Missing the command.')
return result('', 'Error', 'Missing the command.')
invoke_process = subprocess.Popen(self.invoke_cmd,
from utils import CUR_TIME
from utils import logger
from utils import FLAGS
# ================================
# test logging
# ================================
# utils.set_logging(stream=False)
# utils.set_logger(stream=True)
# logger.info("logger111111")
logger.set_logger(stream=True)
logger.info(CUR_TIME)
logger.newline()
logger.error("newline beneath~")
logger.newline(2)
logger.info("haha")
# ================================
# test FLAGS
# ================================
FLAGS.add("--aa", type=float, default=11., help="doc for dd")
logger.info("aa: {}".format(FLAGS.get('aa')))
# for flag that should be overwrite later, don't set default
FLAGS.add("--bb", type=int, default=None, help="doc for dd")
if FLAGS.get('aa') == 11:
FLAGS.overwrite_none(bb=15)
FLAGS.add("--cc", type=bool, default=False, help="doc for dd")
FLAGS.add("--dd", type=str, default="dddddd", help="doc for dd")
# for flag that should be overwrite later, don't set default
FLAGS.add("--ff", type=str, help="doc for dd")
import os
import sys
BASE_DIR = os.path.join(os.path.dirname(__file__), '..')
sys.path.append(BASE_DIR)
from utils.logger import set_logger
logger = set_logger(__name__)
#----------------------------------------------
import numpy as np
#----------------------------------------------
def _get_filepath_vkitti3d_dataset(dataset_path):
"""
VKITTK3Dのデータセットのファイル名を取得する
input:
dataset_path: path/to/vkitti3d_dataset_v1.0/*
output:
ALL_FILES: ex[..., path/to/vkitti3d_dataset_v1.0/06/0020_00500.npy']
"""
import glob
folders = glob.glob(
dataset_path
) #[... , '/Users/washizakikai/data/vkitti3d_dataset_v1.0/06']
ALL_FILES = []
for f in folders:
ALL_FILES += glob.glob(f + "/*")
def main(config, parser):
import utils.logger as logger
(options, args) = parser.parse_args(args=None, values=None)
logConfigDict = logger.set_logger(config['logger'], options.log_path, options.log_level)
loggers = {}
if config['loggers'] and len(config['loggers']):
loggers = load_logger_workers('loggers', config['loggers'], {
'logger_config' : logConfigDict,
'logger_stash' : options.logger_stash,
'batch_rows' : options.batch_rows,
'end_time' : options.end_time,
'end_rowid' : options.end_rowid
})
if len(loggers) > LOGGER_WORKERS_MAX:
elog.error("too many logger workers. please increase LOGGER_WORKERS_MAX and try!")
exit(-1)
found_workers = list_logger_workers(logConfigDict, config['loggers_abspath'])
if options.list_logger_workers:
for logger_worker in found_workers:
elog.info("found worker: %s (%s/%s.py)", logger_worker, config['loggers_abspath'], logger_worker)
elog.force("total %d workers: %r", len(found_workers), found_workers)
return
if options.add_logger:
add_logger(config, found_workers)
return
if options.remove_logger:
remove_logger(config, found_workers)
return
if len(loggers) == 0 and options.force:
loggers = load_logger_workers('loggers', found_workers, {
'logger_config' : logConfigDict,
'logger_stash' : options.logger_stash,
'batch_rows' : options.batch_rows,
'end_time' : options.end_time,
'end_rowid' : options.end_rowid
})
if options.reset_logger_position:
if len(loggers):
reset_logger_position(loggers, config['loggers_abspath'], options.start_time, options.start_rowid)
else:
elog.error("--reset-position ignored: logger worker not found. use --force for all.")
pass
if options.startup:
if len(loggers):
startup(loggers, config)
else:
elog.error("--startup ignored: logger worker not found. use --force for all.")
pass
pass
_TRANS_RESPONSE = {
_MSGTYPE: _TRANS_RESP_TYPE,
_MSGID: "",
_MSGIDRESPONDTO: "",
_MSGFLAG: _FAIL_FLAG,
_TRANSLATIONRESULTS: ""
}
_STATUS_RESPONSE = {
_MSGTYPE: _STA_MSG_TYPE,
_MSGID: "",
_MSGIDRESPONDTO: "",
_MSGFLAG: _HEALTH_FLAG
}
logger = set_logger("api_service.log")
def create_app():
app = Flask(__name__)
app.config.from_object(Params)
@app.route('/', methods=['POST'])
def index():
return json.dumps({"found": "ok"})
@app.route('/translator', methods=['POST'])
def translator():
if request.method == 'POST':
msgType = request.json[_MSGTYPE]
msgID = request.json[_MSGID]
import os
import time
from collections import deque
from utils.logger import set_logger
logger = set_logger()
TRUE = 1
FALSE = 0
UNASSIGN = -1
class FileFormatError(Exception):
""" Raised when file format is not in DIMACS CNF format """
pass
class Solver:
def __init__(self, filename):
logger.info('========= create pysat from %s =========', filename)
self.filename = filename
self.cnf, self.vars = Solver.read_file(filename)
self.learnts = set()
self.assigns = dict.fromkeys(list(self.vars), UNASSIGN)
self.level = 0
self.nodes = dict(
(k, ImplicationNode(k, UNASSIGN)) for k in list(self.vars))
self.branching_vars = set()
self.branching_history = {} # level -> branched variable
self.propagate_history = {} # level -> propagate variables list
self.branching_count = 0
import sys
import datetime
from time import sleep
from multiprocessing import Pool, Value
from itertools import repeat
import pandas as pd
from urllib.request import urlopen
from bs4 import BeautifulSoup
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
from utils.folder import make_folder
from utils.logger import set_logger
logger = set_logger('basic_finance_crawler')
base_path = os.path.dirname(os.path.abspath('..'))
data_path = os.path.join(base_path, 'data')
stock_path = os.path.join(data_path, 'stock')
folder_path = os.path.join(stock_path, 'folder')
summary_path = os.path.join(folder_path, '재무제표_20190331')
make_folder(base_path, data_path, stock_path, folder_path, summary_path)
def get_summary_finance(code):
url = 'http://media.kisline.com/highlight/mainHighlight.nice?nav=1&paper_stock=' + str(
code)
try:
분봉 데이터를 가져오는 코드
"""
import os
import sys
import time
from multiprocessing import Pool, Value
from pykrx import Krx
import pandas as pd
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
from utils.folder import make_folder
from utils.logger import set_logger
logger = set_logger('day_crawler')
base_path = os.path.dirname(os.path.abspath('..'))
data_path = os.path.join(base_path, 'data')
save_path = os.path.join(data_path, 'stock')
folder_path = os.path.join(save_path, 'folder')
day_path = os.path.join(folder_path, '일별매매가격_190414')
make_folder(base_path, data_path, save_path, folder_path, day_path)
krx = Krx()
def get_day_info(code):
df = krx.get_market_ohlcv("20100101", "20190414", code)
df.to_csv(os.path.join(day_path, code + '.csv'))
logger.info("successfully saved " + str(code))
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
from utils.folder import make_folder
from utils.logger import set_logger
base_path = os.path.dirname(os.path.abspath('..'))
data_path = os.path.join(base_path, 'data')
stock_path = os.path.join(data_path, 'stock')
folder_path = os.path.join(stock_path, 'folder')
comment_path = os.path.join(folder_path, '네이버댓글_190414')
code_path = os.path.join(base_path, 'code')
crawler_path = os.path.join(code_path, 'crawler')
make_folder(base_path, data_path, stock_path, folder_path, comment_path, code_path, crawler_path)
logger = set_logger('comment_logger')
krx = Krx()
def get_naver_comment(code):
comments = []
views = []
page = 40
for num in range(1, page):
page = "https://finance.naver.com/item/board.nhn?code=" + str(code) + "&page=" + str(num)
html = urlopen(page)
soup = BeautifulSoup(html, "html.parser")
comment = soup.select('span.tah.gray03')
for i in range(len(comment)):
class parser(ConfigParser.ConfigParser):
def __init__(self, defaults=None):
ConfigParser.ConfigParser.__init__(self, defaults=None)
def optionxform(self, optionstr):
return optionstr
cfg = "%s/default.cfg" % os.path.dirname(os.path.realpath(__file__))
config_raw = parser()
config_raw.read(cfg)
TOKEN = config_raw.get('Kubernetes', 'token_file')
HOSTNAME = socket.gethostname()
logger = logger.set_logger(os.path.basename(__file__), '/var/log/vnclet.log')
class ClientDaemon(CDaemon):
def __init__(self,
name,
save_path,
stdin=os.devnull,
stdout=os.devnull,
stderr=os.devnull,
home_dir='.',
umask=022,
verbose=1):
CDaemon.__init__(self, save_path, stdin, stdout, stderr, home_dir,
umask, verbose)
self.name = name
# Some Arguments Check
assert opt.labeled > 0. and opt.labeled < 1.0
assert opt.unlabeled > 0. and opt.unlabeled <= 1.0
return opt
opt = main()
####################### Output path, logger, device and random seed configuration #################
exp_path = opt.read_model_path if opt.testing else hyperparam_pseudo_method(
opt)
if not os.path.exists(exp_path):
os.makedirs(exp_path)
logger = set_logger(exp_path, testing=opt.testing)
logger.info("Parameters: " + str(json.dumps(vars(opt), indent=4)))
logger.info("Experiment path: %s" % (exp_path))
sp_device, qg_device = set_torch_device(opt.deviceId[0]), set_torch_device(
opt.deviceId[1])
set_random_seed(opt.seed, device='cuda')
################################ Vocab and Data Reader ###########################
sp_copy, qg_copy = 'copy__' in opt.read_sp_model_path, 'copy__' in opt.read_qg_model_path
sp_vocab, qg_vocab = Vocab(opt.dataset, task='semantic_parsing',
copy=sp_copy), Vocab(opt.dataset,
task='question_generation',
copy=qg_copy)
logger.info("Semantic Parsing model vocabulary ...")
logger.info("Vocab size for input natural language sentence is: %s" %
ret[1][0]
])
predictions.append(prediction)
return predictions
if __name__ == "__main__":
args = parse_args()
if args.jobid is not None:
args.run_name = args.run_name + '-' + args.jobid
if args.shortname is None:
args.shortname = args.run_name
np.random.seed(args.seed)
random.seed(args.seed)
logger = set_logger(name=args.shortname, level=args.loglevel)
if args.mode == "train":
logger.info("=> Training mode")
train(args)
else:
logger.info("=> Evaluation mode")
n_features = 35 if args.no_reflex else 36
eval_data, eval_loader = get_eval_dataset(args)
if args.pixor_fusion:
pixor = PixorNet_Fusion(n_features,
groupnorm=args.groupnorm,
resnet_type=args.resnet_type,
image_downscale=args.image_downscale,
resnet_chls=args.resnet_chls)
