def global_r_call(function_name, args=None, r_file=PATIENT_MODULE_FILE_NAME): """ Writing shortcut for all previous function using this same code :param function_name: name of the ro.globalenv function to call :type function_name: str :param args: usually a selector like GroupID :type args: :param r_file: The file to be loaded for the call :type r_file: str :rtype: """ # Source & export R code source_file(r_file) # Export a function to call r_getter_func = ro.globalenv[function_name] # R call data = list() try: # Arguments magics if args: if type(args) is not tuple: args = (args,) data = r_getter_func(*args) else: data = r_getter_func() except RRuntimeError as e: get_logger().error(e) return data
def __init__(self, ml_name, data, split_dict, **kwargs):
self.sample_feats, self.label_feat = self.preprocessing(data.sample_feats, data.label_feats, INDEX)
assert np.all(np.isfinite(self.label_feat))
self.ml_name = ml_name
# training and test
self.Xtrain, self.Xtest, self.ytrain, self.ytest = \
train_test_split(self.sample_feats, self.label_feat, **split_dict)
# model
train_test_str = '-' + dict_to_str(split_dict)
if kwargs is not None and len(kwargs) != 0:
classifier_str = '-' + dict_to_str(kwargs)
pickle_name = str(INDEX) + '-' + self.ml_name + train_test_str + classifier_str + '.pkl'
else:
pickle_name = str(INDEX) + '-' + self.ml_name + train_test_str + '.pkl'
model_pickle = os.path.join(model_dir, pickle_name)
if os.path.exists(model_pickle):
utils.get_logger().warning('{} exists'.format(model_pickle))
with open(model_pickle, 'rb') as model:
self.clf = pickle.load(model)
else:
utils.get_logger().warning('{} does not exist'.format(model_pickle))
self.clf = self.get_clf(kwargs)
self.clf.fit(self.Xtrain, self.ytrain)
with open(model_pickle, 'wb') as model:
pickle.dump(self.clf, model)
def auto_register(mng_cls, cls, proxytype=None, init_args=(),
init_kwargs={}, **kwargs):
'''
Register shared object classes with a default proxytype.
Parameters
----------
cls : type
class which is to be registered with the manager for use
as a shared object
proxytype : subclass of multiprocessing.managers.BaseProxy
Proxy object used to communicate with a shared instance of cls.
If None, then the following steps are attempted:
1) an attempt is made to call the class' build_proxy method which
is expected to provision and return a proxy object as well as
register with the manager any sub-proxies which it expects to
utilize.
2) failing that, a default -> make_inst_proxy(cls) will be used.
'''
assert type(cls) == type
typeid = cls.__name__
if proxytype is None:
try: # to use cls defined proxy
proxytype = cls.build_proxy(mng_cls)
except AttributeError:
proxytype = make_inst_proxy(cls)
get_logger().debug("no proxy was provided for '{}' using "
"default '{}'".format(cls, proxytype))
cls = functools.partial(cls, *init_args, **init_kwargs)
mng_cls.register(typeid, cls, proxytype=proxytype, **kwargs)
def _stop_service(self, container_id):
ship = get_ship_name()
service_dict = None
service_list = kv_list('ships/{}/service/'.format(ship))
if service_list:
key = fnmatch.filter(service_list, '*/{}'.format(container_id))
service_dict = kv_get(key[0]) if key else None
if service_dict and service_dict['Status'] in ['crashed', 'not-recovered']:
kv_remove(key[0])
else:
run_command_in_container('supervisorctl stop armada_agent', container_id)
# TODO: Compatibility with old microservice images. Should be removed in future armada version.
run_command_in_container('supervisorctl stop register_in_service_discovery', container_id)
docker_api = docker_client.api()
last_exception = None
try:
deregister_services(container_id)
except:
traceback.print_exc()
for i in range(3):
try:
docker_api.stop(container_id)
kv_remove(key[0])
except Exception as e:
last_exception = e
traceback.print_exc()
if not is_container_running(container_id):
break
if is_container_running(container_id):
get_logger().error('Could not stop container: {}'.format(container_id))
raise last_exception
def get_event_time(event, epoch=0.0):
'''Return micro-second time stamp value in seconds
'''
value = event.getHeader('Event-Date-Timestamp')
if value is None:
get_logger().warning("Event '{}' has no timestamp!?".format(
event.getHeader("Event-Name")))
return None
return float(value) / 1e6 - epoch
def __init__(self):
if os.path.isfile(data_pickle):
utils.get_logger().warning('{} exists'.format(data_pickle))
with open(data_pickle, 'rb') as raw_data:
self.data = pickle.load(raw_data)
else:
utils.get_logger().warning('{} does not exist'.format(data_pickle))
assert os.path.isfile(data_file)
self.data = np.genfromtxt(data_file, delimiter=',')
with open(data_pickle, 'wb') as raw_data:
pickle.dump(self.data, raw_data)
def connect_callback(timestamp):
parser = argparse.ArgumentParser()
parser.add_argument('--slave-host', help='Hostname or IP of the slave database')
parser.add_argument('--slave-port', help='Port number for the slave database')
parser.add_argument('--slave-user', help='Username for the slave database')
parser.add_argument('--slave-password', help='Password for the slave database')
parser.add_argument('--slave-database', help='Name of the slave database')
parser.add_argument('--history-host', help='Hostname or IP of the history database')
parser.add_argument('--history-port', help='Port number for the history database')
parser.add_argument('--history-user', help='Username for the history database')
parser.add_argument('--history-password', help='Password for the history database')
parser.add_argument('--history-database', help='Name of the history database')
args = parser.parse_args()
slave = {
'host': args.slave_host,
'port': args.slave_port,
'user': args.slave_user,
'password': args.slave_password,
'database': args.slave_database
}
history = {
'host': args.history_host,
'port': args.history_port,
'user': args.history_user,
'password': args.history_password,
'database': args.history_database
}
slavecon, histcon = connect(slave, history)
inspector_logger = get_logger('inspector')
populator_logger = get_logger('populator')
inspector = SlaveInspector(slavecon, logger=inspector_logger)
populator = HistoryPopulator(histcon, logger=populator_logger)
populator.create_tables()
populator.update(timestamp)
for schema in inspector.schemas():
populator.add_schema(schema)
for table in inspector.tables(schema):
inspector.columns(table)
populator.add_table(table)
populator.create_table(table)
populator.fill_table(table)
return slavecon, inspector, populator
def main():
consul_mode, ship_ips, datacenter = _get_runtime_settings()
ship_external_ip = os.environ.get('SHIP_EXTERNAL_IP', '')
consul_config_content = consul_config.get_consul_config(consul_mode, ship_ips, datacenter, ship_external_ip)
with open(consul_config.CONFIG_PATH, 'w') as config_file:
config_file.write(consul_config_content)
command = '/usr/local/bin/consul agent -config-file {config_path}'.format(config_path=consul_config.CONFIG_PATH)
get_logger().info('RUNNING: {}'.format(command))
args = command.split()
os.execv(args[0], args)
def __init__(self, qid, event_queue):
self.logger = get_logger()
self.qid = qid
self.aid = 1003903
self.clientid = random.randrange(11111111, 99999999)
self.msg_id = random.randrange(1111111, 99999999)
self.group_map = {} # 群映射
self.group_m_map = {} # 群到群成员的映射
self.uin_qid_map = {} # uin 到 qq号的映射
self.check_code = None
self.skey = None
self.ptwebqq = None
self.require_check = False
self.QUIT = False
self.last_msg = {}
self.event_queue = event_queue
self.check_data = None # CheckHanlder返回的数据
self.blogin_data = None # 登录前返回的数据
self.rc = 1
self.start_time = time.time()
self.hb_last_time = self.start_time
self.poll_last_time = self.start_time
self._helper = HttpHelper()
self.connected = False
self.polled = False
self.heartbeated = False
self.group_lst_updated = False
def test_rora_connect():
"""
Test if RORA server is online and connection can be made successfully
:return: True|False
:rtype: bool
"""
r_code = 'source("%sconnection.R");' % settings.RORA_LIB
r_code += 'link <- roraConnect();'
r_code += 'dbDisconnect(link);'
try:
ro.r(r_code)
except RRuntimeError as e:
get_logger().error(e)
return False
return True
def extract_from_file(filename, num_process):
import utils
global LOGGER
LOGGER = utils.get_logger()
dataset_path = u'../datasets/wiki'
# get processed words
processed_words = get_target_words(dataset_path)
jobs = dd(list)
for line in codecs.open(filename, encoding='utf-8'):
line = line.split()
target_word, page_title, offset = line[:3]
if target_word not in processed_words:
jobs[target_word].append(dict(word=target_word, page_title=page_title, offset=offset, fetch_links=True))
LOGGER.info("Total {} of jobs available. Num of consumer = {}".format(len(jobs), num_process))
if num_process > 1:
pool = Pool(num_process)
pool.map(extract_instances_for_word, jobs.values())
else:
# for v in jobs.values():
for v in [jobs['milk']]:
extract_instances_for_word(v)
LOGGER.info("Done.")
def proc_luna_running(**kw):
""" Fetch test statuses from Redis, if test finisged notify service via API
and call reduce job.
"""
ext = {'test_id': kw.get('redis_value', {}).get('id')}
logger = get_logger(**ext)
if not ('t_fqdn' in kw and 't_tank_id' in kw):
logger.erro('proc_luna_running call. Malformed params:{}'.format(kw))
try:
except TankClientError as e:
logger.error('Tank API call failed: {}'.format(e))
raise
if tank_msg['status_code'] != 'FINISHED':
if kw.get('status') and TestStateTr.tank_by_port(kw.get('status')) != tank_msg['status_code']:
# test state changes since last check, need to notify
port_state = TestStateTr.port_by_tank(tank_msg['status_code'])
redis_value = kw['redis_value']
redis_value.update({'status': port_state})
redis.hset(r_adr['monitor_finish'], kw['id'],
msgpack.packb(redis_value))
diff = {
'status': port_state,
}
def __init__(self):
my_jid = JID(USER+'/Bot')
self.my_jid = my_jid
settings = XMPPSettings({
"software_name": "qxbot",
"software_version": __version__,
"software_os": "Linux",
"tls_verify_peer": False,
"starttls": True,
"ipv6":False,
"poll_interval": 10,
})
settings["password"] = PASSWORD
version_provider = VersionProvider(settings)
event_queue = settings["event_queue"]
self.webqq = WebQQ(QQ, event_queue)
self.connected = False
#self.mainloop = TornadoMainLoop(settings)
self.mainloop = EpollMainLoop(settings)
self.client = Client(my_jid, [self, version_provider],
settings, self.mainloop)
self.logger = get_logger()
self.msg_dispatch = MessageDispatch(self, self.webqq, BRIDGES)
self.xmpp_msg_queue = Queue.Queue()
def setup():
global Options
global Log
global RootDir, VerifDir
p = argparse.ArgumentParser(
prog='untb',
formatter_class=argparse.RawDescriptionHelpFormatter,
usage="untb <tb_name>",
version="%(prog)s v"+str(__version__),
description=__description__)
p.add_argument('tb_name', action='append', nargs='+', default=None)
p.add_argument('-f', '--force', action='store_true', default=False, help="Force creation of new testbench even if it already exists.")
p.add_argument('-d', '--dbg', action='store_true', default=False, help="Turns on debug lines.")
try:
Options = p.parse_args()
except:
print "Usage error: See untb -h for more information."
sys.exit(1)
verbosity = {False: logging.INFO, True: logging.DEBUG}[Options.dbg]
Log = utils.get_logger('log', verbosity)
utg.Log = Log
try:
RootDir = utils.calc_root_dir()
except utils.AreaError:
Log.critical("CWD is not in an Octeon Tree.")
sys.exit(255)
VerifDir = os.path.join(RootDir, "verif")
def user_network():
"""
Start iteration process for load data to user_network
"""
log = get_logger()
with get_connect() as con, get_connect_iptable() as con_ip:
cur = con.cursor()
cur_ip = con_ip.cursor()
# get settings
settings = get_settings(cur)
load_date_start = datetime.strptime(settings['LOAD_DATE_START'],
'%Y.%m.%d %H:%M:%S.%f')
load_pause = int(settings['LOAD_PAUSE'])
load_pause_empty = int(settings['LOAD_PAUSE_EMPTY'])
cnt_rows = int(settings['LOAD_ROWS'])
while True:
date = user_network_load(cur_ip, cur, load_date_start, cnt_rows)
log.info("Load data. load_date_start: %s", date)
# save date between iterations
if date > load_date_start:
sql = "update load_settings set value=%s where param=%s"
date_str = datetime.strftime(date, '%Y.%m.%d %H:%M:%S.%f')
cur.execute(sql, [date_str, 'LOAD_DATE_START'])
else:
# sleep if not new data
log.debug("No new data sleep( %s )", load_pause_empty)
time.sleep(load_pause_empty)
if load_pause:
log.debug("sleep between loads sleep( %s )", load_pause)
time.sleep(load_pause)
load_date_start = date
def __init__(self, modules):
"""Called when module is loaded"""
self._modules = modules
# Set default name if none is set
if not self.name:
self.name = class_name(self)
# Load logger
self.logger = get_logger(self.name, [stdout])
section = self.get_config(False)
# Load enable
self.enable = None
for key, value in section:
if key.lower() == "enable":
self.enable = value.lower() == "true"
if self.enable is None:
raise Exception("Config for module '%s' doesn't have enable = true/false" % self.name)
# Load interval
self.interval = None
for key, value in section:
if key.lower() == "interval":
self.interval = float(value)
if self.interval is None:
raise Exception("Config for module '%s' doesn't have interval = secs, use 0 for running once" % self.name)
def initialize(self):
"""
Initializes the instance with a mongodb database instance
"""
self.logger = get_logger('profile_handler')
self._connection = get_mongodb_connection()
self._db = self._connection[options.mongodb_name]
def __init__(self, qxbot, webqq, bridges):
self.logger = get_logger()
self.qxbot = qxbot
self.webqq = webqq
self.uin_qid_map = {}
self.qid_uin_map = {}
self.bridges = bridges
self._maped = False
def __init__(self, encoder, decoder, optimizer=RMSprop(), logger=get_logger()):
super(Seq2seq, self).__init__(logger)
self.encoder = encoder
self.decoder = decoder
self.optimizer = optimizer
self.params = self.encoder.params + self.decoder.params
def __init__(self, db):
self.db = db
self.mutex = threading.Lock()
self.job_queue = list()
self.job_queue_sem = threading.Semaphore(0)
self.processing = list()
self.quiting = False
self.logger = utils.get_logger('greedy')
def initialize(self):
"""
Initializes the instance with a mongodn database instance
:param db: an instance to pymongo database object
"""
self._logger = get_logger('lesson_handler')
self._connection = get_mongodb_connection()
self._db = self._connection[options.lesson_db]
def try_group(label_feat):
label_feat = label_feat.astype(np.int8)
uniques = np.unique(label_feat)
if uniques.shape[0] < GROUP_NUM:
return label_feat
utils.get_logger().info('dimension={}, needs fitting'.format(uniques.shape[0]))
fittted_feat = np.zeros(label_feat.shape, dtype=int)
separators = np.linspace(np.min(label_feat), np.max(label_feat), GROUP_NUM)
for i in xrange(label_feat.shape[0]):
for j in xrange(GROUP_NUM - 1):
# print(separators[j], label_feat[i], separators[j+1])
if separators[j] <= label_feat[i] < separators[j + 1]:
fittted_feat[i] = j
break
else:
fittted_feat[i] = GROUP_NUM - 1
return fittted_feat
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--history-host', help='Hostname or IP of the history database')
parser.add_argument('--history-port', help='Port number for the history database')
parser.add_argument('--history-user', help='Username for the history database')
parser.add_argument('--history-password', help='Password for the history database')
parser.add_argument('--history-database', help='Name of the history database')
parser.add_argument('--clone-host', help='Hostname or IP of the clone database')
parser.add_argument('--clone-port', help='Port number for the clone database')
parser.add_argument('--clone-user', help='Username for the clone database')
parser.add_argument('--clone-password', help='Password for the clone database')
parser.add_argument('--clone-database', help='Name of the clone database')
args = parser.parse_args()
history = {
'host': args.history_host,
'port': args.history_port,
'user': args.history_user,
'password': args.history_password,
'database': args.history_database
}
clone = {
'host': args.clone_host,
'port': args.clone_port,
'user': args.clone_user,
'password': args.clone_password,
'database': args.clone_database
}
histcon, clonecon = connect(history, clone)
inspector_logger = get_logger('inspector')
populator_logger = get_logger('populator')
inspector = HistoryInspector(histcon, logger=inspector_logger)
populator = ClonePopulator(clonecon, inspector.update, history, logger=populator_logger)
populator.initialize()
for schema in inspector.schemas():
populator.create_schema(schema)
for table in inspector.tables(schema):
inspector.columns(table)
populator.create_table(table)
clonecon.commit()
def app_root():
"""Base url for steve_zissou service"""
logger = get_logger(__name__)
logger.debug('Request made against root url')
if 'RUN_ENV' in os.environ:
if os.environ['RUN_ENV'] == 'production':
return render_template('base.html')
path_to_js = url_for('static', filename='main.min.js')
return render_template('base.html', path_to_js=path_to_js)
def run():
utils.configure_logger('debug')
logger = utils.get_logger()
input_directory = sys.argv[1] # '../datasets/wiki-filtered'
out_directory = sys.argv[2] # '../datasets/wiki-senses'
files = os.listdir(input_directory)
files = [os.path.join(input_directory, f) for f in files]
logger.info('total number of files: %d' % len(files))
create_sense_dataset(files, out_directory)
logger.info('done')
def run():
utils.configure_logger('debug')
logger = utils.get_logger()
input_directory = sys.argv[1]
out_directory = sys.argv[2]
num_of_fold = int(sys.argv[3])
files = os.listdir(input_directory)
files = [os.path.join(input_directory, f) for f in files]
logger.info('total number of files: %d' % len(files))
create_IMS_formatted_dataset(files, out_directory, k=num_of_fold, num_of_process=30)
logger.info('done')
def reduce_test(test_id, t_fqdn, t_tank_id, eval_only=False):
""" Fetch test statuses from Redis, if test finished notify service via API
and call reduce job.
"""
ext = {'test_id': test_id}
logger = get_logger(**ext)
try:
t_test_id=t_tank_id).get('phout.log')
reduce_phout(test_id, phout_url, eval_only=eval_only)
except TankClientError as e:
logger.error('3rd party API call failed: {}'.format(e))
def _stop_service(self, container_id):
docker_api = docker_client.api()
last_exception = None
try:
exec_id = docker_api.exec_create(container_id, 'supervisorctl stop register_in_service_discovery')
docker_api.exec_start(exec_id['Id'])
except:
traceback.print_exc()
try:
deregister_services(container_id)
except:
traceback.print_exc()
for i in range(3):
try:
docker_api.stop(container_id)
except Exception as e:
last_exception = e
traceback.print_exc()
if not is_container_running(container_id):
break
if is_container_running(container_id):
get_logger().error('Could not stop container: {}'.format(container_id))
raise last_exception
def reduce_arts(test_id, files):
""" Reduce test results without any external API calls,
used to process result manually uploaded by user
without any Lunapark or Jenkins connection.
"""
ext = {'test_id': test_id}
logger = get_logger(**ext)
reduce_phout(test_id, files['phout'])
started_at, finished_at = phout_time_info(files['phout'])
diff = {
'started_at': started_at.isoformat(),
'finished_at': finished_at.isoformat(),
def __init__(self, config, resume_training=True, model_name='dense_bi_lstm'):
# set configurations
self.cfg, self.model_name, self.resume_training, self.start_epoch = config, model_name, resume_training, 1
self.logger = get_logger(os.path.join(self.cfg.ckpt_path, 'log.txt'))
# build model
self._add_placeholder()
self._add_embedding_lookup()
self._build_model()
self._add_loss_op()
self._add_accuracy_op()
self._add_train_op()
print('params number: {}'.format(np.sum([np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()])))
# initialize model
self.sess, self.saver = None, None
self.initialize_session()
args = utils.parse_args()
train_path = args.train
dev_path = args.dev
test_path = args.test
batch_size = args.batch_size
emb_to_use = args.emb
emb_path = args.emb_path
lstm_size = args.num_units
num_filters = args.num_filters
lr = args.lr
decay_rate = args.decay_rate
grad_clip = args.grad_clip
gamma = args.gamma
logger = utils.get_logger("BiLSTM-CNN-CRF")
data = data_utils.load_data(args)
# data["train_word"] = train_data
# data["dev_word"] = dev_data
# data["test_word"] = test_data
# data["word_emb_table"] = word_emb_table
# data["word_collection"] = word_collection
# data["char"] = char_data
# data["label_collection"] = label_collection
# data["train_orth"] = train_data_orth
# data["dev_orth"] = dev_data_orth
# data["test_orth"] = test_data_orth
# data["orth_word_emb_table"] = orth_word_emb_table
# data["orth_char"] = orth_char_data
class WorldDaemon:
""" The world daemon determines whether to update a given world asset (see
assets/world.py) based on the default 'asset_max_age' in settings.cfg or
based on the custom 'max_age' attribute of a given asset.
Since the daemon does not always update all assets, it minimizes resource
usage and can therefore be left running without whaling on CPU and/or
physical memory.
Finally, the world daemon DOES NOT actually refresh/update or otherwise
gather any data or run any queries. Rather, it initializes a World object
(see above) and then works with that object as necessary. """
def __init__(self):
self.logger = utils.get_logger(log_name="world_daemon",
log_level=settings.get(
"world", "log_level"))
self.pid_dir = settings.get("application", "pid_root_dir")
self.pid_file_path = os.path.join(self.pid_dir, "world_daemon.pid")
self.set_pid()
def check_pid_dir(self):
""" Checks to see if the pid directory exists and is writable. Creates a
a new dir if it needs to do so. Also logs a WARN if the user requesting
the check is not the owner of the pid dir. """
if not os.path.isdir(self.pid_dir):
self.logger.error("PID dir '%s' does not exist!" % self.pid_dir)
try:
shutil.os.mkdir(self.pid_dir)
self.logger.critical("Created PID dir '%s'!" % self.pid_dir)
except Exception as e:
self.logger.error("Could not create PID dir '%s'!" %
self.pid_dir)
self.logger.exception(e)
sys.exit(255)
# warn if we're going to start off by trying to write a lock/pid file to
# some other user's directory, b/c that would be bad
pid_dir_owner = getpwuid(os.stat(self.pid_dir).st_uid).pw_name
cur_user = os.getlogin()
if pid_dir_owner != cur_user:
self.logger.warn("PID dir owner is not the current user (%s)!" %
cur_user)
def set_pid(self):
""" Updates 'self.pid' with the int in the daemon pid file. Returns None
if there is no file or the file cannot be parsed. """
self.pid = None
if os.path.isfile(self.pid_file_path):
try:
self.pid = int(file(self.pid_file_path, "rb").read().strip())
except Exception as e:
self.logger.exception(e)
try:
os.kill(self.pid, 0)
except OSError as e:
self.logger.exception(e)
self.logger.error(
"PID %s does not exist! Removing PID file..." % self.pid)
shutil.os.remove(self.pid_file_path)
self.pid = None
def command(self, command=None):
""" Executes a daemon command. Think of this as the router for incoming
daemon commands/operations. Register all commands here. """
if command == "start":
self.start()
elif command == "stop":
self.stop()
elif command == "restart":
self.stop()
time.sleep(3)
self.start()
elif command == "status":
pass
else:
self.logger.error("Unknown daemon command ('%s')!" % command)
# sleep a second and dump a status, regardless of command
time.sleep(1)
self.dump_status()
@retry(
tries=6,
delay=2,
jitter=1,
logger=utils.get_logger(settings.get("world", "log_level")),
)
def start(self):
""" Starts the daemon. """
self.set_pid()
if self.pid is not None:
try:
os.kill(self.pid, 0)
self.logger.warn(
"PID %s exists! Demon is already running! Exiting..." %
self.pid)
sys.exit(1)
except OSError:
self.logger.info("Starting World Daemon...")
else:
self.logger.info("Starting World Daemon...")
# pre-flight sanity checks and initialization tasks
self.check_pid_dir()
context = daemon.DaemonContext(
working_directory=(settings.get("api", "cwd")),
detach_process=True,
umask=0o002,
pidfile=PIDLockFile(self.pid_file_path),
files_preserve=[self.logger.handlers[0].stream],
)
with context:
while True:
try:
self.run()
except Exception as e:
self.logger.error(
"An exception occured during daemonization!")
self.logger.exception(e)
raise
def run(self):
""" A run involves checking all warehouse assets and, if they're older
than their 'max_age' attrib (default to the world.asset_max_age value),
it refreshes them.
Once finished, it sleeps for world.refresh_interval, which is measured
in minutes. """
W = World()
W.refresh_all_assets()
self.logger.debug("World Daemon will sleep for %s minutes..." %
settings.get("world", "refresh_interval"))
time.sleep(settings.get("world", "refresh_interval") * 60)
def stop(self):
""" Stops the daemon. """
self.set_pid()
self.logger.warn("Preparing to kill PID %s..." % self.pid)
if self.pid is not None:
os.kill(self.pid, 15)
time.sleep(2)
try:
os.kill(self.pid, 0)
except:
self.logger.warn("PID %s has been killed." % self.pid)
else:
self.logger.debug(
"Daemon is not running. Ignoring stop command...")
def get_uptime(self, return_type=None):
""" Uses the pid file to determine how long the daemon has been active.
Returns None if the daemon isn't active. Otherwise, this returns a raw
timedelta. """
if os.path.isfile(self.pid_file_path):
pid_file_age = time.time() - os.stat(
self.pid_file_path)[stat.ST_MTIME]
ut = timedelta(seconds=pid_file_age)
uptime = "%sd %sh %sm" % (ut.days, ut.seconds // 3600,
(ut.seconds // 60) % 60)
else:
return None
if return_type == "date":
return datetime.fromtimestamp(
os.stat(self.pid_file_path)[stat.ST_MTIME])
return uptime
def dump_status(self, output_type="CLI"):
""" Prints daemon status to stdout. """
active = False
d = {"active": active}
if self.pid is not None and os.path.isfile(self.pid_file_path):
active = True
if active:
owner_uid = os.stat(self.pid_file_path).st_uid
owner_name = getpwuid(owner_uid).pw_name
try:
utils.mdb.world.find()
except Exception as e:
self.logger.error(
"Daemon is active, but MDB cannot be reached!")
self.logger.exception(e)
raise
d = {}
d["active"] = active
d["up_since"] = self.get_uptime("date")
d["uptime_hms"] = self.get_uptime()
d["owner_uid"] = owner_uid
d["owner_name"] = owner_name
d["pid"] = self.pid
d["pid_file"] = self.pid_file_path
d["assets"] = utils.mdb.world.find().count()
if output_type == dict:
return d
elif output_type == "CLI":
spacer = 15
print("\n\tWorld Daemon stats:\n")
for k, v in sorted(d.iteritems()):
utils.cli_dump(k, spacer, v)
print("\n")
def _extract_axioms(graph, patterns) -> dict:
"""Run actual axiom extraction on CaLiGraph."""
utils.get_logger().debug('CaLi2Ax: Extracting axioms..')
axioms = defaultdict(set)
front_pattern_dict = {}
for (front_pattern,
back_pattern), axiom_patterns in _get_confidence_pattern_set(
patterns, True, False).items():
cat_axioms._fill_dict(
front_pattern_dict,
list(front_pattern), lambda d: cat_axioms._fill_dict(
d, list(reversed(back_pattern)), axiom_patterns))
back_pattern_dict = {}
for (front_pattern,
back_pattern), axiom_patterns in _get_confidence_pattern_set(
patterns, False, True).items():
cat_axioms._fill_dict(
back_pattern_dict,
list(front_pattern), lambda d: cat_axioms._fill_dict(
d, list(reversed(back_pattern)), axiom_patterns))
enclosing_pattern_dict = {}
for (front_pattern,
back_pattern), axiom_patterns in _get_confidence_pattern_set(
patterns, True, True).items():
cat_axioms._fill_dict(
enclosing_pattern_dict,
list(front_pattern), lambda d: cat_axioms._fill_dict(
d, list(reversed(back_pattern)), axiom_patterns))
for node in graph.content_nodes:
property_frequencies = graph.get_property_frequencies(node)
node_labels = set()
for part in graph.get_parts(node):
if cat_util.is_category(part):
node_labels.add(cat_util.category2name(part))
elif list_util.is_listcategory(part) or list_util.is_listpage(
part):
node_labels.add(list_util.list2name(part))
labels_without_by_phrases = [
nlp_util.remove_by_phrase(label, return_doc=True)
for label in node_labels
]
for node_doc in labels_without_by_phrases:
node_axioms = []
front_prop_axiom = _find_axioms(front_pattern_dict, node, node_doc,
property_frequencies)
if front_prop_axiom:
node_axioms.append(front_prop_axiom)
back_prop_axiom = _find_axioms(back_pattern_dict, node, node_doc,
property_frequencies)
if back_prop_axiom:
node_axioms.append(back_prop_axiom)
enclosing_prop_axiom = _find_axioms(enclosing_pattern_dict, node,
node_doc, property_frequencies)
if enclosing_prop_axiom:
node_axioms.append(enclosing_prop_axiom)
prop_axioms_by_pred = {
a[1]: {x
for x in node_axioms if x[1] == a[1]}
for a in node_axioms
}
for pred, similar_prop_axioms in prop_axioms_by_pred.items():
if dbp_store.is_object_property(pred):
res_labels = {
a[2]: dbp_store.get_label(a[2])
for a in similar_prop_axioms
}
similar_prop_axioms = {
a
for a in similar_prop_axioms
if all(res_labels[a[2]] == val
or res_labels[a[2]] not in val
for val in res_labels.values())
}
best_prop_axiom = max(similar_prop_axioms,
key=operator.itemgetter(3))
axioms[node].add(best_prop_axiom)
utils.get_logger().debug(
f'CaLi2Ax: Extracted {sum(len(axioms) for axioms in axioms.values())} axioms for {len(axioms)} categories.'
)
return axioms
# -*- coding: utf-8 -*-
"""
Created by Dr.W on 2018/5/16
"""
import os
import config
from utils import get_yam
from utils import set_time
from utils import get_logger
from utils.phone import Phone
from entity.tasks import Tasks
from multiprocessing import Pool
from utils.appium import AppiumServer
log = get_logger("runner")
def selector_devices(ds):
res = []
for d in ds.get("appium", []):
if d.get("closed", False):
log.info("this device is closed.")
continue
res.append(d)
return res
def get_devices():
ds = get_yam(config.DEVICES_PATH)
return selector_devices(ds)
parser.add_argument(
"--ocm-url",
default=
"https://api-integration.6943.hive-integration.openshiftapps.com")
parser.add_argument("--ocp-versions")
parser.add_argument("--installation-timeout", type=int)
parser.add_argument("--public-registries", default="")
parser.add_argument("--img-expr-time", default="")
parser.add_argument("--img-expr-interval", default="")
parser.add_argument("--check-cvo", default="False")
return deployment_options.load_deployment_options(parser)
deploy_options = handle_arguments()
log = utils.get_logger('deploy-service-configmap')
SRC_FILE = os.path.join(os.getcwd(), 'deploy/assisted-service-configmap.yaml')
DST_FILE = os.path.join(os.getcwd(), 'build', deploy_options.namespace,
'assisted-service-configmap.yaml')
SERVICE = "assisted-service"
def get_deployment_tag(args):
if args.deploy_manifest_tag:
return args.deploy_manifest_tag
if args.deploy_tag:
return args.deploy_tag
def main():
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================
import tensorflow as tf
from spatial_transformer3 import transformer
import numpy as np
from tf_utils import weight_variable, bias_variable, dense_to_one_hot
import cv2
from resnet import output_layer
import get_data
from config import *
import time
from tensorflow.contrib.slim.nets import resnet_v2
slim = tf.contrib.slim
import utils
logger = utils.get_logger()
def get_4_pts(theta, batch_size):
with tf.name_scope('get_4_pts'):
pts1_ = []
pts2_ = []
pts = []
h = 2.0 / grid_h
w = 2.0 / grid_w
tot = 0
one_ = tf.ones([batch_size, 1, 2]) / do_crop_rate
temp_p = [
0.5, 0.5, 0, 0.5, -0.5, 0.5, 0.5, 0, 0.3, 0.3, -0.5, 0, 0.5, -0.5,
0, -0.5, -0.5, -0.5
]
def main():
config = Config()
logger, log_dir = get_logger(os.path.join(config.model, "logs/"))
logger.info("=======Model Configuration=======")
logger.info(config.desc)
logger.info("=================================")
try:
train_x, dev_x, test_x, train_y, dev_y, test_y, train_m, dev_m, test_m, test_dt = load_agg_selected_data_mem(data_path=config.data_path, \
x_len=config.x_len, \
y_len=config.y_len, \
foresight=config.foresight, \
cell_ids=config.train_cell_ids, \
dev_ratio=config.dev_ratio, \
test_len=config.test_len, \
seed=config.seed)
model = Model(config)
if config.allow_gpu:
model = model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)
# get train data
TrainDataSet = BatchDataset(train_x, train_m, train_y)
TrainSampler = tud.RandomSampler(TrainDataSet)
TrainDataLoader = tud.DataLoader(TrainDataSet,
batch_size=config.batch_size,
sampler=TrainSampler,
num_workers=2)
# get valid Data
dev_x, dev_m, dev_y = torch.Tensor(dev_x), torch.Tensor(
dev_m), torch.Tensor(dev_y)
if config.allow_gpu:
dev_x, dev_m, dev_y = dev_x.cuda(), dev_m.cuda(), dev_y.cuda()
step = 0
no_improv = 0
best_loss = 100
model_dir = make_date_dir(os.path.join(config.model, 'model_save/'))
logger.info("Start training")
start_time = time()
for i in range(config.num_epochs):
epoch = i + 1
# train
model.train()
for batch_x, batch_m, batch_y in TrainDataLoader:
step = step + 1
if config.allow_gpu:
batch_x, batch_m, batch_y = batch_x.cuda(), batch_m.cuda(
), batch_y.cuda()
optimizer.zero_grad()
prediction, loss, rse, smape, mae = model(
batch_x, batch_m, batch_y)
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(
model.parameters(), config.clip)
optimizer.step()
if step % 100 == 0:
logger.info(
"epoch: %d, step: %d, loss: %.4f, rse: %.4f, smape: %.4f, mae: %.4f"
% (epoch, step, loss, rse, smape, mae))
# dev score for each epoch (no mini batch)
with torch.no_grad():
model.eval()
prediction, dev_loss, dev_rse, dev_smape, dev_mae = model(
dev_x, dev_m, dev_y)
if dev_loss < best_loss:
best_loss = dev_loss
no_improv = 0
# logger.info("New score! : dev_loss: %.4f, dev_rse: %.4f, dev_smape: %.4f, dev_mae: %.4f" %
# (dev_loss, dev_rse, dev_smape, dev_mae))
# logger.info("Saving model at {}".format(model_dir))
torch.save(model, model_dir + "/" + config.model + ".pth")
else:
no_improv += 1
if no_improv == config.nepoch_no_improv:
logger.info("No improvement for %d epochs" % no_improv)
break
elapsed = time() - start_time
# generating results (no mini batch)
logger.info("Saving model at {}".format(model_dir))
logger.info("Elapsed training time {0:0.2f} mins".format(elapsed / 60))
logger.info("Training finished, exit program")
except:
logger.exception("ERROR")
import utils
if __name__ == '__main__':
print(id(utils.get_logger()))
print(id(utils.get_logger()))
import pydicom as dicom
from os.path import join as pjoin
import os
import sys
import numpy as np
from dicom_utils import get_patient_data
from utils import get_logger
logger = get_logger(__name__)
class SaleReader:
def __init__(self, path):
self.path = path
self.images = []
self.num_images = 0
self.dcm_paths = []
self.patient_data = None
self.broken = False
self._read_images()
def _read_images(self):
# find .dcm files
files = [f for f in os.listdir(self.path) if f.lower().endswith('.dcm')]
if len(files) == 0:
self.broken = True
logger.warning('There are no frames. This folder should be deleted. Path: {}'.format(self.path))
return
slice_sequences_by_imagetype_by_seriesdesc = \
self._read_slice_sequences(files)
import numpy as np
import scipy.special
import time
from edge_selection_tree import evaluate_accept_reject_outcome
from kidney_ip import solve_picef_model
from utils import get_logger, query_node_hash, outcome_node_hash
logger = get_logger()
LARGE_UCB_CONST = 1000
def create_mcts_policy():
def mcts_get_next_edge(graph, queried_edges, edge_rejections, tree):
# prepare the tree by resetting the root and removing unneeded query nodes.
accepted_edges = [
e for e, r in zip(queried_edges, edge_rejections) if not r
]
rejected_edges = [
e for e, r in zip(queried_edges, edge_rejections) if r
]
new_root = MultistageOutcomeNode(None, accepted_edges, rejected_edges,
tree)
tree.prepare_tree(new_root)
best_child = tree.search_next_child()
if best_child is None:
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(),
FLAGS.emb_file,
list(itertools.chain.from_iterable(
[[w[0] for w in s] for s in test_sentences])
)
)
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(
train_sentences, char_to_id, tag_to_id, FLAGS.lower
)
dev_data = prepare_dataset(
dev_sentences, char_to_id, tag_to_id, FLAGS.lower
)
test_data = prepare_dataset(
test_sentences, char_to_id, tag_to_id, FLAGS.lower
)
print("%i / %i / %i sentences in train / dev / test." % (
len(train_data), len(dev_data), len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(FLAGS.max_epoch):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def train(args):
utils.make_workspace_dirs(args.workspace)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
anchors = np.loadtxt(args.anchors) if args.anchors else None
# Shared size between collate_fn and scale sampler.
shared_size = torch.IntTensor(args.in_size).share_memory_()
scale_sampler = utils.TrainScaleSampler(shared_size, args.scale_step,
args.rescale_freq)
logger = utils.get_logger(path=os.path.join(args.workspace, 'log.txt'))
torch.backends.cudnn.benchmark = True
dataset = ds.HotchpotchDataset(args.dataset_root,
cfg='./data/train.txt',
backbone=args.backbone,
augment=True)
collate_fn = partial(ds.collate_fn, in_size=shared_size)
data_loader = torch.utils.data.DataLoader(dataset,
args.batch_size,
True,
num_workers=args.workers,
collate_fn=collate_fn,
pin_memory=args.pin,
drop_last=True)
num_ids = int(dataset.max_id + 1)
if args.backbone == 'darknet':
model = darknet.DarkNet(anchors,
num_classes=args.num_classes,
num_ids=num_ids).to(device)
elif args.backbone == 'shufflenetv2':
model = shufflenetv2.ShuffleNetV2(anchors,
num_classes=args.num_classes,
num_ids=num_ids,
model_size=args.thin,
box_loss=args.box_loss,
cls_loss=args.cls_loss).to(device)
elif args.backbone == 'sosnet':
model = sosmot.SOSMOT(anchors,
num_classes=args.num_classes,
num_ids=num_ids,
box_loss=args.box_loss,
cls_loss=args.cls_loss).to(device)
else:
print('unknown backbone architecture!')
sys.exit(0)
if args.checkpoint:
model.load_state_dict(torch.load(args.checkpoint))
params = [p for p in model.parameters() if p.requires_grad]
backbone_neck_params, detection_params, identity_params = grouping_model_params(
model)
if args.optim == 'sgd':
# optimizer = torch.optim.SGD(params, lr=args.lr,
# momentum=args.momentum, weight_decay=args.weight_decay)
optimizer = torch.optim.SGD([{
'params': backbone_neck_params
}, {
'params': detection_params,
'lr': args.lr * args.lr_coeff[1]
}, {
'params': identity_params,
'lr': args.lr * args.lr_coeff[2]
}],
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
else:
optimizer = torch.optim.Adam(params,
lr=args.lr,
weight_decay=args.weight_decay)
if args.freeze_bn:
for name, param in model.named_parameters():
if 'norm' in name:
param.requires_grad = False
logger.info('freeze {}'.format(name))
else:
param.requires_grad = True
trainer = f'{args.workspace}/checkpoint/trainer-ckpt.pth'
if args.resume:
trainer_state = torch.load(trainer)
optimizer.load_state_dict(trainer_state['optimizer'])
if -1 in args.milestones:
args.milestones = [int(args.epochs * 0.5), int(args.epochs * 0.75)]
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
optimizer, milestones=args.milestones, gamma=args.lr_gamma)
start_epoch = 0
if args.resume:
start_epoch = trainer_state['epoch'] + 1
lr_scheduler.load_state_dict(trainer_state['lr_scheduler'])
logger.info(args)
logger.info('Start training from epoch {}'.format(start_epoch))
model_path = f'{args.workspace}/checkpoint/{args.savename}-ckpt-%03d.pth'
for epoch in range(start_epoch, args.epochs):
model.train()
logger.info(('%8s%10s%10s' + '%10s' * 8) %
('Epoch', 'Batch', 'SIZE', 'LBOX', 'LCLS', 'LIDE', 'LOSS',
'SBOX', 'SCLS', 'SIDE', 'LR'))
rmetrics = defaultdict(float)
optimizer.zero_grad()
for batch, (images, targets) in enumerate(data_loader):
warmup = min(args.warmup, len(data_loader))
if epoch == 0 and batch <= warmup:
lr = args.lr * (batch / warmup)**4
for i, g in enumerate(optimizer.param_groups):
g['lr'] = lr * args.lr_coeff[i]
loss, metrics = model(images.to(device), targets.to(device),
images.shape[2:])
loss.backward()
if args.sparsity:
model.correct_bn_grad(args.lamb)
num_batches = epoch * len(data_loader) + batch + 1
if ((batch + 1) % args.accumulated_batches
== 0) or (batch == len(data_loader) - 1):
optimizer.step()
optimizer.zero_grad()
for k, v in metrics.items():
rmetrics[k] = (rmetrics[k] * batch + metrics[k]) / (batch + 1)
fmt = tuple([('%g/%g') % (epoch, args.epochs), ('%g/%g') % (batch,
len(data_loader)), ('%gx%g') % (shared_size[0].item(), shared_size[1].item())] + \
list(rmetrics.values()) + [optimizer.param_groups[0]['lr']])
if batch % args.print_interval == 0:
logger.info(('%8s%10s%10s' + '%10.3g' *
(len(rmetrics.values()) + 1)) % fmt)
scale_sampler(num_batches)
torch.save(model.state_dict(), f"{model_path}" % epoch)
torch.save(
{
'epoch': epoch,
'optimizer': optimizer.state_dict(),
'lr_scheduler': lr_scheduler.state_dict()
}, trainer)
if epoch >= args.eval_epoch:
pass
lr_scheduler.step()
parser.add_argument("--exp", type=int, default=0, help="experiment id number")
args = parser.parse_args()
args.mem = args.recurrence > 1
# Define run dir
suffix = datetime.datetime.now().strftime("%y-%m-%d-%H-%M-%S")
default_model_name = "{}_{}_seed{}_{}".format(args.env, args.algo, args.seed,
suffix)
model_name = args.model or default_model_name
model_dir = utils.get_model_dir(args.env, model_name, args.exp, args.seed)
# Define logger, CSV writer, json args, and Tensorboard writer
logger = utils.get_logger(model_dir)
csv_file, csv_writer = utils.get_csv_writer(model_dir)
if args.tb:
from tensorboardX import SummaryWriter
tb_writer = SummaryWriter(model_dir)
with open(model_dir + '/args.json', 'w') as outfile:
json.dump(vars(args), outfile)
# Log command and all script arguments
logger.info("{}\n".format(" ".join(sys.argv)))
logger.info("{}\n".format(args))
# Set seed for all randomness sources
utils.seed(args.seed)
import sys, string, os
import json, pickle
import numpy as np
from collections import defaultdict
from data_utils import *
import gensim, utils
MAX_LENGTH = 120
MAX_CHAR_PER_WORD = 45
root_symbol = "##ROOT##"
root_label = "<ROOT>"
word_end = "##WE##"
logger = utils.get_logger("LoadData")
def read_conll_sequence_labeling(path,
word_alphabet,
label_alphabet,
train_abble=True,
out_dir=None):
"""
read data from file in conll format
:param path: file path
:return: sentences of words and labels, sentences of indexes of words and labels.
"""
#word_alphabet = []
#label_alphabet = ['O', "PER", "MISC", "ORG", "LOC"]
word_sentences = []
label_sentences = []
word_index_sentences = []
}
save_path = os.path.join(
cfg["training"]["checkpoint"],
"{}_{}_best_model.pkl".format(cfg["model"]["arch"],
cfg["data"]["dataset"]),
)
torch.save(state, save_path)
epoch += 1
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="config")
parser.add_argument(
"--config",
nargs="?",
type=str,
default="configs/fcn8s_pascal.yml",
help="Configuration file to use",
)
args = parser.parse_args()
with open(args.config) as fp:
cfg = yaml.load(fp, Loader=yaml.Loader)
logdir = cfg["training"]["checkpoint"]
logger = utils.get_logger(logdir)
shutil.copy(args.config, logdir)
train(cfg, logger)
import time
from watchdog.observers import Observer as WatchObserver
from watchdog.events import FileSystemEventHandler
from utils import RABBITMQ_HOST, get_logger
from kombu import Connection, Queue, uuid, Consumer, Producer
import difflib
import signal
log = get_logger()
class Observer():
def __init__(self):
self.worker = None
signal.signal(signal.SIGTERM, self.exit)
signal.signal(signal.SIGINT, self.exit)
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
def run(self):
observer = WatchObserver()
dirname = '/etc/artemis'
filename = 'config.yaml'
try:
with Connection(RABBITMQ_HOST) as connection:
event_handler = self.Handler(dirname, filename, connection)
observer.schedule(event_handler, dirname, recursive=False)
observer.start()
log.info('started')
self.should_stop = False
timestamp = str(int(time.time())) if args.mode == 'train' else args.demo_model
output_path = os.path.join('.', args.train_data + "_save", timestamp)
if not os.path.exists(output_path): os.makedirs(output_path)
summary_path = os.path.join(output_path, "summaries")
paths['summary_path'] = summary_path
if not os.path.exists(summary_path): os.makedirs(summary_path)
model_path = os.path.join(output_path, "checkpoints")
if not os.path.exists(model_path): os.makedirs(model_path)
ckpt_prefix = os.path.join(model_path, "model")
paths['model_path'] = ckpt_prefix
result_path = os.path.join(output_path, "results")
paths['result_path'] = result_path
if not os.path.exists(result_path): os.makedirs(result_path)
log_path = os.path.join(result_path, "log.txt")
paths['log_path'] = log_path
get_logger(log_path).info(str(args))
if __name__ == '__main__':
build_word_index(args.word_embedding_file, args.src_vocab_file,
args.tgt_file, args.tgt_vocab_file)
src_vocab = get_vocab(args.src_vocab_file)
src_vocab_size = len(src_vocab)
src_unknown = src_vocab_size
src_padding = src_vocab_size + 1
#print(len(src_vocab))
#print(vocab_size)
tgt_vocab = get_vocab(args.tgt_vocab_file)
tgt_vocab_size = len(tgt_vocab)
import argparse
import os
import yaml
import deployment_options
import utils
parser = argparse.ArgumentParser()
deploy_options = deployment_options.load_deployment_options(parser)
log = utils.get_logger('deploy-image-service')
SRC_FILE = os.path.join(os.getcwd(), 'deploy/assisted-image-service.yaml')
DST_FILE = os.path.join(os.getcwd(), 'build', deploy_options.namespace,
'assisted-image-service.yaml')
def main():
utils.verify_build_directory(deploy_options.namespace)
with open(SRC_FILE, "r") as src:
log.info(
f"Loading source template file for assisted-image-service: {SRC_FILE}"
)
raw_data = src.read()
raw_data = raw_data.replace('REPLACE_NAMESPACE',
f'"{deploy_options.namespace}"')
raw_data = raw_data.replace('REPLACE_IMAGE_SERVICE_IMAGE',
os.environ.get("IMAGE_SERVICE"))
data = yaml.safe_load(raw_data)
log.info(data)
import utils
from models.augment_cnn import AugmentCNN
from torch.utils.data import Dataset, DataLoader
import scipy.io as io
from thop import profile
from thop import clever_format
config = AugmentConfig()
device = torch.device("cuda")
# tensorboard
writer = SummaryWriter(log_dir=os.path.join(config.path, "tb"))
writer.add_text('config', config.as_markdown(), 0)
logger = utils.get_logger(os.path.join(config.path, "{}.log".format(config.name)))
config.print_params(logger.info)
def main():
logger.info("Logger is set - training start")
fileRoot = r'/home/hlu/Data/VIPL'
saveRoot = r'/home/hlu/Data/VIPL_STMap' + str(config.fold_num) + str(config.fold_index)
n_classes = 1
input_channels = 3
input_size = np.array([64, 300])
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
toTensor = transforms.ToTensor()
resize = transforms.Resize(size=(64, 300))
# set default gpu device id
import argparse
from utils import get_logger
import pandas as pd
from utils import read_file_contents_list
import numpy as np
import os
import matplotlib.pyplot as plt
logger = get_logger('Dice')
def get_dice_data_table(dice_data_root, item_list, num_scans, mask_flag):
data_table = np.zeros((num_scans, len(item_list)))
for item_idx in range(len(item_list)):
item = item_list[item_idx]
dice_csv_path = os.path.join(dice_data_root, item)
dice_csv_path = os.path.join(dice_csv_path, mask_flag + '.csv')
logger.info(f'Read dice csv: {dice_csv_path}')
dice_df = pd.read_csv(dice_csv_path)
np_list = dice_df['Dice'].to_numpy()
data_table[:, item_idx] = np_list[:]
return data_table
def main():
parser = argparse.ArgumentParser('Box plot for dice statistics.')
parser.add_argument('--dice-data-root', type=str)
parser.add_argument('--item-list', type=str)
parser.add_argument('--out-fig-folder', type=str)
parser.add_argument('--num-scans', type=int, default=50)
args = parser.parse_args()
--region= Snapshot region (default: current region)
--size= Image rootfs size (default: snapshot size)
--name= Image name (default: snapshot name, ie. $turnkey_version.ebs)
--desc= Image website link (default: enumerated from name)
--arch= Image architecture (default: enumerated from name)
"""
import sys
import getopt
import utils
from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping
log = utils.get_logger('ebs-register')
class Error(Exception):
pass
def usage(e=None):
if e:
print >> sys.stderr, "error: " + str(e)
print >> sys.stderr, "Syntax: %s [ -options ] snapshot_id" % (sys.argv[0])
print >> sys.stderr, __doc__.strip()
sys.exit(1)
import pandas as pd
import os
import subprocess
from parse import parse
from fit_circle import fit_circle_algebraic
import matplotlib.pyplot as plt
from matplotlib import colors
from utils import mkdir_p
import nibabel as nib
from scipy import ndimage as ndi
import skimage.morphology
import skimage.measure
import argparse
import SimpleITK as sitk
logger = get_logger('Inference the CT native padding mask')
def create_body_mask(in_img, out_mask):
rBody = 2
print(f'Get body mask of image {in_img}')
image_itk = sitk.ReadImage(in_img)
gaussian_filter = sitk.DiscreteGaussianImageFilter()
gaussian_filter.SetVariance(2)
print(f'Apply gaussian filter')
filtered_image = gaussian_filter.Execute(image_itk)
image_np = sitk.GetArrayFromImage(filtered_image)
from model import Model
import torch
from hyperparams import hps
from data_batcher import Vocab, restore_text
from utils import (get_logger, DEBUG, tonp, Timer)
logger = get_logger(__name__, DEBUG)
def train():
logger.info('start building vocab data')
vocab = Vocab(hps.vocab_file, hps.vocab_size)
logger.info('end building vocab data')
logger.info('vocab size: %s' % vocab.size())
model = Model(vocab, hps)
if hps.use_cuda:
model = model.cuda()
if hps.restore is not None:
# raise ValueError('Noe data to restore')
model.load_state_dict(torch.load(hps.restore))
logger.info('----Start training----')
timer = Timer()
timer.start()
for step in range(hps.start_step, hps.num_iters + 1):
# Forward -------------------------------------------------------------
outputs = model(None, None, 'infer')
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# 创建相应文件夹
make_path(FLAGS)
# 配置文件读取
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
# 初始化Flask，并且渲染静态页面
app = Flask(__name__, static_url_path="/static")
# 加载配置文件
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
# 配置GPU
tf_config = tf.ConfigProto()
sess = tf.Session(config=tf_config)
# 加载训练好的模型
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config,
id_to_char, logger)
def heartbeat():
print(
time.strftime('%Y-%m-%d %H:%M:%S - heartbeat',
time.localtime(time.time())))
timer = threading.Timer(60, heartbeat)
timer.start()
""" Module for handling data requests from the client. """
from flask_socketio import emit
from database import get_cities, get_objects_in_sector
from global_context import CITIES, RESOURCE_NODES
from models import City, ResourceNode, Ship, Tile
from utils import get_logger
from world.coordinates import Coordinate
from server import current_user, database_session, login_required, socketio
LOGGER = get_logger(__name__)
@socketio.on("get_cities")
@login_required
def send_cities(message): # pylint: disable=unused-argument
""" Trigger a 'load_city' event for each city in the game. """
LOGGER.debug(f"City list requested by {current_user}")
for city in get_cities(database_session,
current_user.ship.location.sector):
resources = {}
for resource_held in city.resources:
resources[resource_held.resource.name] = resource_held.amount
response = {
"position": city.location.coordinate,
"id": city.id,
"name": city.name,
"resources": resources,
"population": city.population,
from pyppeteer_spider.spider import PyppeteerSpider
from utils import short_sleep, get_logger
from datetime import datetime
from pprint import pformat
from pathlib import Path
import aioredis
import asyncio
import json
import csv
import sys
logger = get_logger(logger_name="pinterest",
log_save_path=Path("../logs/pinterest.log"))
redis_client = None
prof_data_key = 'pinterest_prof_data'
proc_dir_pages = 'pinterest_processed_dir_pages'
data_file_header = ['name','username','url','location','bio','posts','followerCount','followingCount','avatar_url','domain','crawled','status_code']
file_stem = 'pinterest_user_data'
file_rows = 0
data_dir = Path(f"../data/{file_stem}")
data_dir.mkdir(exist_ok=True,parents=True)
# find current file for storing profile data.
data_file_numbers = [int(f.stem.split('.')[-1]) for f in data_dir.glob("*.tsv")]
if len(data_file_numbers) > 1:
file_idx = max(data_file_numbers)
#!/usr/bin/env python3.6
from os import environ, system
from sys import exit, argv
from utils import filebot, categories, get_logger
from config import config
from cleaner import clean
logger = get_logger('PlexBotDeluge')
logger.info("Starting PlexBotTorrent...")
base = config.torrents.strip('"')
torrent_id = argv[1]
torrent_name = argv[2]
torrent_dir = argv[3]
logger.info("Torrent ID: " + torrent_id)
if torrent_dir == base + "/Movies":
category = "movies"
elif torrent_dir == base + "/Shows":
category = "tv"
else:
syslog("Invalid Torrent")
exit(0)
cat = categories(category)
args = cat + " --torrent"
logger.info("FileBot Args: " + args)
filebot(args, None, False)
clean(torrent_id);
logger.info("Done :():")
exit(0)
next_state[2] = self.value_table[str(state)]
if col != self.width - 1:
next_state[3] = self.value_table[str([col + 1, row])]
else:
next_state[3] = self.value_table[str(state)]
return next_state
# 메인 함수
if __name__ == "__main__":
# 0 이면 every-visit
# 1 이면 first-visit
logger = get_logger('mc.log')
visit = [1]
env = Env()
EveryVisit0_FirstVisit1 = 1
agent = MCAgent(actions=list(range(env.n_actions)))
success_cnt = 0
fail_cnt = 0
total_step = 0
MAX_EPISODES = 1000 # 최대 에피소드 수
for episode in range(MAX_EPISODES):
state = env.reset() # 에피소드 시작 : 환경을 초기화하고, 상태 = 초기상태로 설정
action = agent.get_action(state)
def __init__(self,
writer,
log_interval,
num_of_children,
input_channels,
input_shape,
num_classes,
learning_rate_child,
momentum_child,
num_branches,
num_of_layers,
out_filters,
controller_size,
controller_layers,
t0,
eta_min,
t_mult,
epoch_child,
isShared,
eps_clip=0.2,
K_epochs=10,
path=""):
self.writer = writer
self.log_interval = log_interval
self.path = path
self.num_of_children = num_of_children
self.K_epochs = K_epochs
self.input_shape = input_shape
self.input_channels = input_channels
self.num_classes = num_classes
self.out_filters = out_filters
self.isShared = isShared
self.epoch_child = epoch_child
self.num_branches = num_branches
self.num_layers = num_of_layers
self.learning_rate_child = learning_rate_child
self.momentum = momentum_child
self.t0 = t0
self.eta_min = eta_min
self.t_mult = t_mult
self.eps_clip = eps_clip
self.controller = PPOController(
self.writer, self.num_layers, self.num_branches, controller_size,
controller_layers
) # self,num_layers=2,num_branches=4,lstm_size=5,lstm_num_layers=2,tanh_constant=1.5,temperature=None
self.controller_old = PPOController(
self.writer, self.num_layers, self.num_branches, controller_size,
controller_layers
) # self,num_layers=2,num_branches=4,lstm_size=5,lstm_num_layers=2,tanh_constant=1.5,temperature=None
self.controller.load_state_dict(self.controller_old.state_dict())
self.children = list()
self.bestchilds = Kindergarden(best_of=3)
self.globaliter = 0
self.logger = get_logger()
if self.isShared:
self.child = SharedEnasChild(self.num_layers,
self.learning_rate_child,
self.momentum,
num_classes=self.num_classes,
out_filters=self.out_filters,
input_shape=self.input_shape,
input_channels=self.input_channels,
t0=t0,
eta_min=eta_min)
import aiohttp
import json
import asyncio
import re
from bs4 import BeautifulSoup
import sys
import os
sys.path.append(os.pardir)
from utils import get_proxy, get_ua, start_request, get_logger
logger = get_logger('jingxi_goods_analysis')
async def make_headers(session):
ua = await get_ua(session)
headers = {'user-agent': ua}
return headers
class GoodsAnalysis:
def __init__(self):
self.data = []
async def get_page_data(self, session, sku_id):
url = f'https://m.jingxi.com/item/view?sku={sku_id}'
for i in range(3):
headers = await make_headers(session)
proxy = await get_proxy(session)
res = await start_request(session,
