def demo():
"""
Demo.
"""
print("SprintDataset demo.")
from argparse import ArgumentParser
from Util import progress_bar_with_time
from Log import log
from Config import Config
from Dataset import init_dataset
arg_parser = ArgumentParser()
arg_parser.add_argument("--config", help="config with ExternSprintDataset", required=True)
arg_parser.add_argument("--sprint_cache_dataset", help="kwargs dict for SprintCacheDataset", required=True)
arg_parser.add_argument("--max_num_seqs", default=sys.maxsize, type=int)
arg_parser.add_argument("--action", default="compare", help="compare or benchmark")
args = arg_parser.parse_args()
log.initialize(verbosity=[4])
sprint_cache_dataset_kwargs = eval(args.sprint_cache_dataset)
assert isinstance(sprint_cache_dataset_kwargs, dict)
sprint_cache_dataset = SprintCacheDataset(**sprint_cache_dataset_kwargs)
print("SprintCacheDataset: %r" % sprint_cache_dataset)
config = Config()
config.load_file(args.config)
dataset = init_dataset(config.typed_value("train"))
print("Dataset via config: %r" % dataset)
assert sprint_cache_dataset.num_inputs == dataset.num_inputs
assert tuple(sprint_cache_dataset.num_outputs["classes"]) == tuple(dataset.num_outputs["classes"])
sprint_cache_dataset.init_seq_order(epoch=1)
if args.action == "compare":
print("Iterating through dataset...")
seq_idx = 0
dataset.init_seq_order(epoch=1)
while seq_idx < args.max_num_seqs:
if not dataset.is_less_than_num_seqs(seq_idx):
break
dataset.load_seqs(seq_idx, seq_idx + 1)
tag = dataset.get_tag(seq_idx)
assert not tag.startswith("seq-"), "dataset does not provide tag-names for seqs"
dataset_seq = sprint_cache_dataset.get_dataset_seq_for_name(tag)
data = dataset.get_data(seq_idx, "data")
targets = dataset.get_data(seq_idx, "classes")
assert data.shape == dataset_seq.features["data"].shape
assert targets.shape == dataset_seq.features["classes"].shape
assert numpy.allclose(data, dataset_seq.features["data"])
assert numpy.allclose(targets, dataset_seq.features["classes"])
seq_idx += 1
progress_bar_with_time(dataset.get_complete_frac(seq_idx))
print("Finished through dataset. Num seqs: %i" % seq_idx)
print("SprintCacheDataset has num seqs: %i." % sprint_cache_dataset.num_seqs)
elif args.action == "benchmark":
print("Iterating through dataset...")
start_time = time.time()
seq_tags = []
seq_idx = 0
dataset.init_seq_order(epoch=1)
while seq_idx < args.max_num_seqs:
if not dataset.is_less_than_num_seqs(seq_idx):
break
dataset.load_seqs(seq_idx, seq_idx + 1)
tag = dataset.get_tag(seq_idx)
assert not tag.startswith("seq-"), "dataset does not provide tag-names for seqs"
seq_tags.append(tag)
dataset.get_data(seq_idx, "data")
dataset.get_data(seq_idx, "classes")
seq_idx += 1
progress_bar_with_time(dataset.get_complete_frac(seq_idx))
print("Finished through dataset. Num seqs: %i, time: %f" % (seq_idx, time.time() - start_time))
print("SprintCacheDataset has num seqs: %i." % sprint_cache_dataset.num_seqs)
if hasattr(dataset, "exit_handler"):
dataset.exit_handler()
else:
print("No way to stop any background tasks.")
del dataset
start_time = time.time()
print("Iterating through SprintCacheDataset...")
for i, tag in enumerate(seq_tags):
sprint_cache_dataset.get_dataset_seq_for_name(tag)
progress_bar_with_time(float(i) / len(seq_tags))
print("Finished through SprintCacheDataset. time: %f" % (time.time() - start_time,))
else:
raise Exception("invalid action: %r" % args.action)
def __init__(self, archFile):
from Config import Config
self.Cfg = Config(archFile)
#build regex for data entries
types = '|'.join(self.Cfg['DataTypes'].keys())
self.dataEntryRegex = re.compile(
"^\s*([^\s:]+)\s*:\s+\.(" + types +
")\s+((\s*[^\s,]+\s*,)*\s*[^\s,]+)\s*$")
#regex of valid operations per instructiontype
operationsPerType = {}
for instructionTypeName, instructionList in self.Cfg.instructions.items(
):
operationsPerType[instructionTypeName] = "|".join([
instDesciption['@mnemonic']
for _, instDesciption in instructionList.items()
])
#construct list of regex for each instructionType
self.instructionRegexes = {}
for instructionTypeName, instructionType in self.Cfg['ISA'][
'instructiontypes'].items():
self.instructionRegexes[instructionTypeName] = []
for fieldName, field in instructionType.items():
if fieldName[0] == '@':
continue #this is an attribute, not a sub_field in the xml config
fieldTypeName = field['@type']
fieldType = self.Cfg['ISA']['instructionFieldTypes'][
fieldTypeName]
#opcode is a special field, allowed values are constructed from the instruction names of the various classes
if fieldTypeName == "mnemonic":
self.instructionRegexes[instructionTypeName] += [
('mnemonic',
"(" + operationsPerType[instructionTypeName] + ")",
"(" + fieldType["@separator"] + "|$)")
]
#the 'unused' field is not for the parser, skip it
elif fieldTypeName == "unused":
continue
#add the regex to the list
else:
if '@regex' not in fieldType:
raise ValueError('instructionFieldTypes ' +
fieldTypeName +
' does not specify a regex')
self.instructionRegexes[instructionTypeName] += [
(fieldName, fieldType['@regex'],
"(" + fieldType["@separator"] + "|$)")
]
#remove trailing separator from last field in instruction
lastRegex = self.instructionRegexes[instructionTypeName][-1]
self.instructionRegexes[instructionTypeName][-1] = (lastRegex[0],
lastRegex[1],
"$")
#add regex for label
self.label_token = '(\$[^\s;:]+)'
self.instructionRegexes['label'] = [('target', self.label_token, ""),
(None, '(:)', "$")]
self.function_label_token = 'def\s+(\w+)\s*:'
self.instructionRegexes['function_label'] = [
('target', self.function_label_token, "")
]
import os
import time
import utils
import numpy as np
import pandas as pd
import tensorflow as tf
from LSTM_CRF_PA import LSTM_CRF_PA
from Config import Config
print "preparing data"
con = Config()
(X_char_train,
y_train), (X_char_dev,
y_dev), (X_char_test,
y_test), (X_char_PA, y_PA) = utils.get_AllData(con.maxlen)
char2id, id2char = utils.loadMap(con.map_dict['char2id'])
label2id, id2label = utils.loadMap(con.map_dict['label2id'])
num_chars = len(id2char.keys())
num_classes = len(id2label.keys())
print 'num of chars:', num_chars
print 'num of classes:', num_classes
#X_char_train, y_train = utils.get_PA_data(con.dataset['traindata'], char2id, label2id)
# merge export and PA train data
X_char_merge_train, y_merge_train = utils.merge_export_and_PA_train_data(
original script was obtained from cheiner-chemistry library (MIT lisence).
some codes were changed
#TODO: we used GGNN, but conventional models, such as CNN, may be used to input "adjacency matrix" + "node vector".
"""
import chainer
from chainer import cuda
from chainer import datasets
import chainer.links as L
import chainer.functions as F
from GGNN import GGNN
from Config import Config
#settings
CF = Config()
CATEGORY_DIM = CF.CATEGORY_DIM
VALUE_DIM = CF.VALUE_DIM
if CF.genre == ["V"]:
OUT_DIM = 1
else:
OUT_DIM = VALUE_DIM
#GPU mode
GPUMode = CF.GPUMode
if GPUMode:
xp = cuda.cupy
import cupy
gpu_device = 0
# get image scale
img_scale = img.shape[0], img.shape[1]
# print(img.shape)
img, bbox, label = self.transforms(img, bbox, label)
# from (h, w, c) to (c, h, w)
img = torch.tensor(img).permute(2, 0, 1)
# from bgr to rgb
img = img[(2, 1, 0), :, :]
return img, bbox, label, img_id, ignore, img_scale
def __len__(self):
return len(self.id_list)
if __name__ == "__main__":
config = Config("local")
train_dataset = VOC_dataset(
config.voc2007_root,
config.voc2012_root,
config.voc2007_test_anno,
trn_val="test",
)
# just check everything is fine
for img, bbox, label in tqdm(train_dataset):
pass
print(len(train_dataset))
from Directions import Directions
from Models import *
my_logger = logging.getLogger('traffic-logger')
my_logger.setLevel(logging.DEBUG)
handler = logging.FileHandler('logs/traffic-logger.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
my_logger.addHandler(handler)
try:
config = Config().get()
engine = create_engine(config['sql_connection_string'])
Session = sessionmaker(bind=engine)
session = Session()
api_key = config['google_api_key']
forward_route_id = 1
backward_route_id = 2
forward_directions = Directions(session, api_key, forward_route_id)
backward_directions = Directions(session, api_key, backward_route_id)
my_logger.addHandler(handler)
import os
import time
import numpy as np
from numpy import random
import pandas as pd
import logging
import pdb
import tempfile
from sqlalchemy import create_engine
from sklearn.preprocessing import MinMaxScaler
from Config import Config
conf = Config()
log = logging.getLogger(__name__)
log.info('%s logger started.',__name__)
def load_data(instrument, train):
if train:
data_path = conf.TRAINING_DATA_PATH
csv_path = os.path.join(data_path, instrument + conf.csv_file)
if conf.num_of_rows_read > 0:
return pd.read_csv(csv_path, sep=';', nrows=conf.num_of_rows_read)
else:
return pd.read_csv(csv_path, sep=';')
else:
data_path = conf.INPUT_PREDICT_DATA_PATH
def main(_):
loader = Loader(flag="azenuz_small")
config = Config(loader, flag="azenuz_small")
config.gpu = 1
if platform.system() == 'Linux':
gpuid = config.gpu
os.environ["CUDA_VISIBLE_DEVICES"] = '{}'.format(gpuid)
device = '/gpu:' + str(gpuid)
else:
device = '/cpu:0'
lr_updater = LearningRateUpdater(config.learning_rate, config.decay, config.decay_epoch)
i = 0
graph = tf.Graph()
with graph.as_default():
trainm = CTR_ggnn(config, device, loader, "Train")
testm = CTR_ggnn(config, device, loader, "Valid")
session_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
session_config.gpu_options.allow_growth = True
with tf.Session(graph=graph, config=session_config) as session:
session.run(tf.global_variables_initializer())
# CTR_GNN_loader(session, config)
best_auc = 0.
best_logloss = 1.
best_epoch_auc = 0.
best_epoch_logloss = 0.
auc = 0.
for epoch in range(config.epoch_num):
trainm.update_lr(session, lr_updater.get_lr())
cost, auc = run(session, config, trainm, loader, verbose=True)
INFO_LOG("Epoch %d Train AUC %.3f" % (epoch + 1, auc))
INFO_LOG("Epoch %d Train costs %.3f" %
(epoch, cost))
session.run(tf.local_variables_initializer())
cost, auc = run(session, config, testm, loader, verbose=True)
INFO_LOG("Epoch %d Valid AUC %.3f" % (epoch, auc))
INFO_LOG("Epoch %d Valid cost %.3f" % (epoch, cost))
# #
lr_updater.update(auc, epoch)
if best_auc < auc:
best_auc = auc
best_epoch_auc = epoch
CTR_GNN_saver(session, config, best_auc, best_epoch_auc)
if best_logloss > cost:
best_logloss = cost
best_epoch_logloss = epoch
# CTR_GNN_saver(session, config, best_epoch_logloss, best_epoch_logloss)
INFO_LOG("*** best AUC now is %.3f in %d epoch" % (best_auc, best_epoch_auc))
INFO_LOG("*** best logloss now is %.3f in %d epoch" % (best_logloss, best_epoch_logloss))
if epoch % 300 == 0 and epoch != 0:
loader.change_data_list(loader.increase_data_idx())
def __init__(self):
self.config = Config()
"""The driver program; imports other modules, accepts
command-line arguments and calls upon other modules to display a fractal
on-screen and write a PNG image. This file is the main entry point of the
program."""
import sys
from FractalFactory import FractalFactory
from GradientFactory import GradientFactory
from ImagePainter import ImagePainter
from Config import Config
if __name__ == '__main__':
if len(sys.argv) < 2:
configDict = Config().readFile(None)
ImagePainter(FractalFactory().makeFractal(configDict),
GradientFactory().makeGradient(configDict, None),
configDict)
elif len(sys.argv) < 3:
configDict = Config().readFile(sys.argv[1])
ImagePainter(FractalFactory().makeFractal(configDict),
GradientFactory().makeGradient(configDict, None),
configDict)
else:
configDict = Config().readFile(sys.argv[1])
ImagePainter(FractalFactory().makeFractal(configDict),
GradientFactory().makeGradient(configDict, sys.argv[2]),
configDict)
def __init__(self, stage_table):
self.db_config = Config().db_config
#self.insert_query = Config().insert_query
self.query = SqlExecute(stage_table)
def __init__(self):
self.config = Config()
self._srcdir = self.config.src_dir
self._prefix = self.config.prefix
def reset(self):
self.config = Config()
def main():
config = Config()
config.setArgsConfig(parseScriptArguments(sys.argv[1:]))
if config.argsconfig['task'] is Task.DETERMINEMODEL:
for filename in listOfModelTestFiles(
Config().argsconfig['inputDirOrFile']):
generator = BonnmotionParamsGenerator()
generator.setModeltestFilename(filename)
generator.createBonnmotionParamsFiles()
if config.argsconfig['delete'] == True:
DataAccess().cleartable(generator.modelname)
log("starting model-determination of " + generator.modelname)
md = ModelDeterminationDispatcher(generator.noOfFilesCreated)
log("done. " + str(generator.noOfFilesCreated) + " hashes saved.")
md.cleanup()
elif config.argsconfig['task'] is Task.VALIDATEMODEL:
md5 = []
sha1 = []
result = []
if config.argsconfig.has_key('arg'):
DataAccess().get(config.argsconfig['arg'], result, md5,
sha1) #get parameters and hashes from database
log("starting model-validation of " + config.argsconfig['arg'])
else:
DataAccess().get4(result, md5, sha1)
n = 0
for x in result:
f = open(
os.path.join(
Config().readConfigEntry('bonnmotionvalidatepath'),
config.readConfigEntry('tempoutputparamsfile').replace(
'INDEX', str(n))), 'w')
f.write(x['bmparamsfile'])
f.close()
n += 1
mv = ModelValidationDispatcher(n, md5, sha1)
log("done. " + str(n) + " hashes checked.")
elif config.argsconfig['task'] is Task.DETERMINEAPP:
for filename in listOfAppTestFiles(
config.argsconfig['inputDirOrFile']):
generator = BonnmotionParamsGenerator()
generator.setApptestFilename(filename)
params = generator.parseApptestFile()
if config.argsconfig['delete'] == True:
DataAccess().cleartable(params['app']) ##APP-DATATABLE
if params['inputDirOrFile'] is InputDirOrFile.FILE:
generator2 = BonnmotionParamsGenerator()
generator2.setModeltestFilename(params['paramsfile'])
generator2.createBonnmotionParamsFiles()
log("starting app determination: " + filename +
", modeltest: " + params['paramsfile'])
bmd = BonnmotionDispatcher(
generator2.noOfFilesCreated,
config.readConfigEntry('bonnmotionvalidatepath'))
add = AppDeterminationDispatcher(generator2.noOfFilesCreated,
params)
log("done")
add.cleanup()
elif params['inputDirOrFile'] is InputDirOrFile.DIRECTORY:
for file in os.listdir(params['paramsfile']):
if file.endswith('.modeltest'):
generator2 = BonnmotionParamsGenerator()
generator2.setModeltestFilename(
os.path.join(params['paramsfile'], file))
generator2.createBonnmotionParamsFiles()
log("starting app determination: " + filename +
", modeltest: " + file)
bmd = BonnmotionDispatcher(
generator2.noOfFilesCreated,
config.readConfigEntry('bonnmotionvalidatepath'))
add = AppDeterminationDispatcher(
generator2.noOfFilesCreated, params)
add.cleanup()
log("done")
elif config.argsconfig['task'] is Task.VALIDATEAPP:
result = []
if config.argsconfig.has_key('arg'):
DataAccess().get2(config.argsconfig['arg'],
result) #get parameters and hashes from database
log("starting app-validation of " + config.argsconfig['arg'])
else:
DataAccess().get3(result)
n = 0
for x in result:
f = open(
os.path.join(
Config().readConfigEntry('bonnmotionvalidatepath'),
config.readConfigEntry('tempoutputparamsfile').replace(
'INDEX', str(n))), 'w')
f.write(x['bmparamsfile'])
f.close()
n += 1
bmd = BonnmotionDispatcher(
n, config.readConfigEntry('bonnmotionvalidatepath'))
AppValidationDispatcher(n, result).cleanup()
log("done. " + str(n) + " hashes checked.")
def __init__(self):
self.config = Config()
self.data = DataLoaders()
def __init__(self, config_filename):
self.config = Config(config_filename=config_filename)
def init_config(config_filename=None,
command_line_options=(),
default_config=None,
extra_updates=None):
"""
:param str|None config_filename:
:param list[str]|tuple[str] command_line_options: e.g. ``sys.argv[1:]``
:param dict[str]|None default_config:
:param dict[str]|None extra_updates:
Initializes the global config.
There are multiple sources which are used to init the config:
* ``configFilename``, and maybe first item of ``commandLineOptions`` interpret as config filename
* other options via ``commandLineOptions``
* ``extra_updates``
Note about the order/priority of these:
* ``extra_updates``
* options from ``commandLineOptions``
* ``configFilename``
* config filename from ``commandLineOptions[0]``
* ``extra_updates``
* options from ``commandLineOptions``
``extra_updates`` and ``commandLineOptions`` are used twice so that they are available
when the config is loaded, which thus has access to them, and can e.g. use them via Python code.
However, the purpose is that they overwrite any option from the config;
that is why we apply them again in the end.
``commandLineOptions`` is applied after ``extra_updates`` so that the user has still the possibility
to overwrite anything set by ``extra_updates``.
"""
global config
config = Config()
config_filenames_by_cmd_line = []
if command_line_options:
# Assume that the first argument prefixed with "+" or "-" and all following is not a config file.
i = 0
for arg in command_line_options:
if arg[:1] in "-+":
break
config_filenames_by_cmd_line.append(arg)
i += 1
command_line_options = command_line_options[i:]
if default_config:
config.update(default_config)
if extra_updates:
config.update(extra_updates)
if command_line_options:
config.parse_cmd_args(command_line_options)
if config_filename:
config.load_file(config_filename)
for fn in config_filenames_by_cmd_line:
config.load_file(fn)
if extra_updates:
config.update(extra_updates)
if command_line_options:
config.parse_cmd_args(command_line_options)
# I really don't know where to put this otherwise:
if config.bool("EnableAutoNumpySharedMemPickling", False):
import TaskSystem
TaskSystem.SharedMemNumpyConfig["enabled"] = True
# Server default options
if config.value('task', 'train') == 'server':
config.set('num_inputs', 2)
config.set('num_outputs', 1)
def fetch_remote_list(args, require_attribs=False, recursive=None):
def _get_filelist_remote(remote_uri, recursive=True):
## If remote_uri ends with '/' then all remote files will have
## the remote_uri prefix removed in the relative path.
## If, on the other hand, the remote_uri ends with something else
## (probably alphanumeric symbol) we'll use the last path part
## in the relative path.
##
## Complicated, eh? See an example:
## _get_filelist_remote("s3://bckt/abc/def") may yield:
## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} }
## _get_filelist_remote("s3://bckt/abc/def/") will yield:
## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} }
## Furthermore a prefix-magic can restrict the return list:
## _get_filelist_remote("s3://bckt/abc/def/x") yields:
## { 'xyz/blah.txt' : {} }
info(u"Retrieving list of remote files for %s ..." % remote_uri)
s3 = S3(Config())
response = s3.bucket_list(remote_uri.bucket(),
prefix=remote_uri.object(),
recursive=recursive)
rem_base_original = rem_base = remote_uri.object()
remote_uri_original = remote_uri
if rem_base != '' and rem_base[-1] != '/':
rem_base = rem_base[:rem_base.rfind('/') + 1]
remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base))
rem_base_len = len(rem_base)
rem_list = SortedDict(ignore_case=False)
break_now = False
for object in response['list']:
if object['Key'] == rem_base_original and object['Key'][
-1] != os.path.sep:
## We asked for one file and we got that file :-)
key = os.path.basename(object['Key'])
object_uri_str = remote_uri_original.uri()
break_now = True
rem_list = {
} ## Remove whatever has already been put to rem_list
else:
key = object['Key'][
rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !!
object_uri_str = remote_uri.uri() + key
rem_list[key] = {
'size': int(object['Size']),
'timestamp': dateS3toUnix(
object['LastModified']
), ## Sadly it's upload time, not our lastmod time :-(
'md5': object['ETag'][1:-1],
'object_key': object['Key'],
'object_uri_str': object_uri_str,
'base_uri': remote_uri,
}
if break_now:
break
return rem_list
cfg = Config()
remote_uris = []
remote_list = SortedDict(ignore_case=False)
if type(args) not in (list, tuple):
args = [args]
if recursive == None:
recursive = cfg.recursive
for arg in args:
uri = S3Uri(arg)
if not uri.type == 's3':
raise ParameterError("Expecting S3 URI instead of '%s'" % arg)
remote_uris.append(uri)
if recursive:
for uri in remote_uris:
objectlist = _get_filelist_remote(uri)
for key in objectlist:
remote_list[key] = objectlist[key]
else:
for uri in remote_uris:
uri_str = str(uri)
## Wildcards used in remote URI?
## If yes we'll need a bucket listing...
if uri_str.find('*') > -1 or uri_str.find('?') > -1:
first_wildcard = uri_str.find('*')
first_questionmark = uri_str.find('?')
if first_questionmark > -1 and first_questionmark < first_wildcard:
first_wildcard = first_questionmark
prefix = uri_str[:first_wildcard]
rest = uri_str[first_wildcard + 1:]
## Only request recursive listing if the 'rest' of the URI,
## i.e. the part after first wildcard, contains '/'
need_recursion = rest.find('/') > -1
objectlist = _get_filelist_remote(S3Uri(prefix),
recursive=need_recursion)
for key in objectlist:
## Check whether the 'key' matches the requested wildcards
if glob.fnmatch.fnmatch(objectlist[key]['object_uri_str'],
uri_str):
remote_list[key] = objectlist[key]
else:
## No wildcards - simply append the given URI to the list
key = os.path.basename(uri.object())
if not key:
raise ParameterError(
u"Expecting S3 URI with a filename or --recursive: %s"
% uri.uri())
remote_item = {
'base_uri': uri,
'object_uri_str': unicode(uri),
'object_key': uri.object()
}
if require_attribs:
response = S3(cfg).object_info(uri)
remote_item.update({
'size':
int(response['headers']['content-length']),
'md5':
response['headers']['etag'].strip('"\''),
'timestamp':
dateRFC822toUnix(response['headers']['date'])
})
remote_list[key] = remote_item
return remote_list
#判断路径是否存在，否则创建
def check_path(filename):
#将文件路径分割出来
file_dir = os.path.split(filename)[0]
if not os.path.isdir(file_dir):
os.makedirs(file_dir)
return 1 #表示路径不存在，已创建文件
return 2 #表示路径已存在
'''
全局参数
'''
#获取路径
my_dirpath = cur_file_dir()
# my_log = Logger(os.path.join(dirpath, "./log/service%s.log"%(time.strftime("%Y-%m-%d_%H%M", time.localtime()) )),level='debug')
#日志路径
log_path = os.path.join(my_dirpath, "./log/service")
check_path(log_path)
my_log = Logger(log_path, level='debug')
#获取配置
config_path = os.path.join(my_dirpath, "config.ini")
print(config_path)
config = Config(config_path)
my_secret, svc_name, svc_display_name, my_debug, my_version = config.get_config(
)
my_log.logger.info(
[my_secret, svc_name, svc_display_name, my_debug, my_version])
def compare_filelists(src_list, dst_list, src_remote, dst_remote):
def __direction_str(is_remote):
return is_remote and "remote" or "local"
# We don't support local->local sync, use 'rsync' or something like that instead ;-)
assert (not (src_remote == False and dst_remote == False))
info(u"Verifying attributes...")
cfg = Config()
exists_list = SortedDict(ignore_case=False)
debug("Comparing filelists (direction: %s -> %s)" %
(__direction_str(src_remote), __direction_str(dst_remote)))
debug("src_list.keys: %s" % src_list.keys())
debug("dst_list.keys: %s" % dst_list.keys())
for file in src_list.keys():
debug(u"CHECK: %s" % file)
if dst_list.has_key(file):
## Was --skip-existing requested?
if cfg.skip_existing:
debug(u"IGNR: %s (used --skip-existing)" % (file))
exists_list[file] = src_list[file]
del (src_list[file])
## Remove from destination-list, all that is left there will be deleted
del (dst_list[file])
continue
attribs_match = True
## Check size first
if 'size' in cfg.sync_checks and dst_list[file][
'size'] != src_list[file]['size']:
debug(u"XFER: %s (size mismatch: src=%s dst=%s)" %
(file, src_list[file]['size'], dst_list[file]['size']))
attribs_match = False
if attribs_match and 'md5' in cfg.sync_checks:
## ... same size, check MD5
try:
if src_remote == False and dst_remote == True:
src_md5 = hash_file_md5(src_list[file]['full_name'])
dst_md5 = dst_list[file]['md5']
elif src_remote == True and dst_remote == False:
src_md5 = src_list[file]['md5']
dst_md5 = hash_file_md5(dst_list[file]['full_name'])
elif src_remote == True and dst_remote == True:
src_md5 = src_list[file]['md5']
dst_md5 = dst_list[file]['md5']
except (IOError, OSError), e:
# MD5 sum verification failed - ignore that file altogether
debug(u"IGNR: %s (disappeared)" % (file))
warning(u"%s: file disappeared, ignoring." % (file))
del (src_list[file])
del (dst_list[file])
continue
if src_md5 != dst_md5:
## Checksums are different.
attribs_match = False
debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" %
(file, src_md5, dst_md5))
if attribs_match:
## Remove from source-list, all that is left there will be transferred
debug(u"IGNR: %s (transfer not needed)" % file)
exists_list[file] = src_list[file]
del (src_list[file])
## Remove from destination-list, all that is left there will be deleted
del (dst_list[file])
import torch
from torch.utils.data import DataLoader
from MyDataSet import MyDataSet
from Tokenizer import Tokenizer
from pad import pad
from Config import Config
from Seq2Seq import Seq2Seq
if __name__ == '__main__':
source_path = '../data/test/source.txt'
target_path = '../data/test/target.txt'
vocab_path = '../data/vocab.txt'
model_path = '../model/model.pth'
tokenizer = Tokenizer(vocab_path)
config = Config()
fr = open('../result/test.txt', 'w', encoding='utf-8') # 存储预测结果
loader = DataLoader(dataset=MyDataSet(source_path, target_path, tokenizer),
batch_size=config.batch_size,
shuffle=True,
num_workers=2,
collate_fn=pad,
drop_last=False) # 最后一个batch数据集不丢弃
if not torch.cuda.is_available():
print('No cuda is available!')
exit()
device = torch.device('cuda:0')
model = Seq2Seq(config)
model.to(device)
def fetch_local_list(args, recursive=None):
def _get_filelist_local(local_uri):
info(u"Compiling list of local files...")
if local_uri.isdir():
local_base = deunicodise(local_uri.basename())
local_path = deunicodise(local_uri.path())
filelist = _fswalk(local_path, cfg.follow_symlinks)
single_file = False
else:
local_base = ""
local_path = deunicodise(local_uri.dirname())
filelist = [(local_path, [], [deunicodise(local_uri.basename())])]
single_file = True
loc_list = SortedDict(ignore_case=False)
for root, dirs, files in filelist:
rel_root = root.replace(local_path, local_base, 1)
for f in files:
full_name = os.path.join(root, f)
if not os.path.isfile(full_name):
continue
if os.path.islink(full_name):
if not cfg.follow_symlinks:
continue
relative_file = unicodise(os.path.join(rel_root, f))
if os.path.sep != "/":
# Convert non-unix dir separators to '/'
relative_file = "/".join(relative_file.split(os.path.sep))
if cfg.urlencoding_mode == "normal":
relative_file = replace_nonprintables(relative_file)
if relative_file.startswith('./'):
relative_file = relative_file[2:]
sr = os.stat_result(os.lstat(full_name))
loc_list[relative_file] = {
'full_name_unicode': unicodise(full_name),
'full_name': full_name,
'size': sr.st_size,
'mtime': sr.st_mtime,
## TODO: Possibly more to save here...
}
return loc_list, single_file
cfg = Config()
local_uris = []
local_list = SortedDict(ignore_case=False)
single_file = False
if type(args) not in (list, tuple):
args = [args]
if recursive == None:
recursive = cfg.recursive
for arg in args:
uri = S3Uri(arg)
if not uri.type == 'file':
raise ParameterError(
"Expecting filename or directory instead of: %s" % arg)
if uri.isdir() and not recursive:
raise ParameterError("Use --recursive to upload a directory: %s" %
arg)
local_uris.append(uri)
for uri in local_uris:
list_for_uri, single_file = _get_filelist_local(uri)
local_list.update(list_for_uri)
## Single file is True if and only if the user
## specified one local URI and that URI represents
## a FILE. Ie it is False if the URI was of a DIR
## and that dir contained only one FILE. That's not
## a case of single_file==True.
if len(local_list) > 1:
single_file = False
return local_list, single_file
def __init__(self):
self.compressorConfig = Config().compressorConfig
self.imageHandlerConfig = Config().imageHandlerConfig
args = parser.parse_args()
if not os.path.isdir(args.proj_dir):
print "!! Please provide a valid directory, given: %s" % (args.proj_dir)
sys.exit()
print "Going to take snapshot for project %s" % (args.proj_dir)
print "Creating output directory at %s" % (args.out_dir)
Util.cleanup(args.log_file)
Log.setLogger(args.verbose, args.log_file)
cfg = Config(args.config_file)
#1. First, retrieve the 1st commit date from SQL server
commit_dates = fetchCommitDates(cfg, args.proj_dir, args.lang)
#2. Snapshot
dumpSnapShots(args.proj_dir, args.out_dir, int(args.ss_interval_len), commit_dates[0], commit_dates[1])
project_name = pathLeaf(args.proj_dir)
ss_dir = os.path.abspath(args.out_dir)
ss_names = os.listdir(ss_dir + '/' + project_name)
ss_names.sort()
ss_paths = [ss_dir + '/' + project_name + '/' + ss_name + '/' for ss_name in ss_names]
ss_name_to_sha = {}
for ss_index, ss_path in enumerate(ss_paths):
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 4 16:20:39 2017
@author: Yacalis
"""
import time
from Config import Config
# =============================================================================
# the 'regular' format combines [lat, lon, month, time] into one dimension,
# but the 'detailed' format leaves all of that uncombined
# =============================================================================
if Config().config.use_detailed_data:
nc_file = '../SPCAM/SPCAM_Stephan/SPCAM_outputs_detailed.nc'
mean_file = '../SPCAM/SPCAM_Stephan/SPCAM_mean_detailed.nc'
std_file = '../SPCAM/SPCAM_Stephan/SPCAM_std_detailed.nc'
else:
nc_file = '../SPCAM/SPCAM_Pierre/SPCAM_outputs_tropics.nc'
mean_file = '../SPCAM/SPCAM_new_norms/SPCAM_mean.nc'
std_file = '../SPCAM/SPCAM_new_norms/SPCAM_std.nc'
max_file = '../SPCAM/SPCAM_new_norms/SPCAM_max.nc'
# =============================================================================
# the logdir name is long, but it beats having to look at the parameter json
# file just to see what the most important values are
# =============================================================================
def get_logdir(config: object) -> str:
:param recv: 邮箱接收人地址，多个账号以逗号隔开
:param title: 邮件标题
:param content: 邮件内容
:param mail_host: 邮箱服务器
:param port: 端口号
:return:
'''
if file:
msg = MIMEMultipart()
# 构建正文
part_text = MIMEText(content)
msg.attach(part_text) # 把正文加到邮件体里面去
# 构建邮件附件
part_attach1 = MIMEApplication(open(file, 'rb').read()) # 打开附件
part_attach1.add_header('Content-Disposition', 'attachment', filename=file) # 为附件命名
msg.attach(part_attach1) # 添加附件
else:
msg = MIMEText(content) # 邮件内容
msg['Subject'] = title # 邮件主题
msg['From'] = username # 发送者账号
msg['To'] = recv # 接收者账号列表
smtp = smtplib.SMTP(mail_host, port=port)
smtp.login(username, passwd) # 登录
smtp.sendmail(username, recv, msg.as_string())
smtp.quit()
if __name__ == '__main__':
from Config.Config import *
path = Config().case_data_file_path
send_mail(username='******',passwd='XKXCOUSQLDJJLWRP',recv='*****@*****.**',title='python_test',content='python_test_report',file=path)
class Keyframe:
kfID = 0
voc = pyfbow.Vocabulary()
voc.readFromFile(Config().bow)
kfdb = KeyframeDatabase()
def __init__(self, frame):
self.timestamp = frame.timestamp
self.kfID = Keyframe.kfID
self.fID = frame.id
Keyframe.kfID += 1
self.fp, self.des = frame.fp, frame.des
self.depth = frame.depth
self.height, self.width = frame.height, frame.width
self.pose = frame.pose
# self.rot = frame.rot
# self.pos = frame.pos
self.neighbors = []
# self.bow = []
# self.bow_ind = []
# def compute_bow(self):
# Extract bow vector from features, and the indices of the words at the 4th lvl of tree
self.bow, bow_ind = Keyframe.voc.transform2(self.des, 4)
self.bow_ind = bow_ind.keys()
Keyframe.kfdb.insert(self)
self.key_points = dict()
self.n_kp = 0 # Number of keypoints
self.covariance = None
def key_point_initializer(self):
fx = Config().fx
fy = Config().fy
cx = Config().cx
cy = Config().cy
for idx in range(len(self.fp)):
d = self.depth[idx]
pos = [(self.fp[idx].pt[0] - cx) / fx * d,
(self.fp[idx].pt[1] - cy) / fy * d, d]
self.key_points[idx] = KeyPoint(pos, self.des[idx])
self.n_kp = len(self.fp)
def add_key_point(self, idx, kp): # idx of the feature point, and kp
if idx not in self.key_points: # if kp is not in this
self.key_points[idx] = kp
self.n_kp += 1
return True
else:
assert kp.id == self.key_points[idx].id # .get_id()
return False
def new_key_point(self, idx, pos):
if idx not in self.key_points:
new_kp = KeyPoint(pos, self.des[idx])
self.key_points[idx] = new_kp
self.n_kp += 1
return new_kp
else:
return self.key_points[idx]
def get_key_point(self, idx):
if idx in self.key_points:
return self.key_points[idx]
else:
return None
def set_pose(self, rot, trans):
pose = np.eye(4)
pose[0:3, 0:3] = rot
pose[0:3, 3] = trans
self.pose = pose
def rot(self):
# if self.pose:
# return Rotation.from_quat(self.pose[3:]).as_matrix()
return self.pose[0:3, 0:3]
# return None
def pos(self):
# if self.pose:
# return np.array(self.pose[0:3])
return self.pose[0:3, 3]
# return None
def pose_matrix(self):
# r = self.rot()
# t = self.pos()
# tt = np.expand_dims(t, axis=1)
# return np.concatenate((np.concatenate((r, tt), axis=1),
# np.array([0, 0, 0, 1], ndmin=2)), axis=0)
return self.pose
def neighbors_list(self):
return [kf for kf, _, _ in self.neighbors]
import gym
import gym_battleships
from stable_baselines.common.vec_env import DummyVecEnv
from stable_baselines.common.callbacks import CheckpointCallback, EvalCallback, StopTrainingOnRewardThreshold
from stable_baselines import ACKTR
from Config import Config
from Result import Result
# Inits Battleship gym environments and config
config = Config(5, [3, 2, 2], True, False, False)
env2 = gym.make('Battleships-v0', config=config)
env3 = gym.make('Battleships-v0', config=config)
env = DummyVecEnv([lambda: env2])
env4 = DummyVecEnv([lambda: env3])
# Define Callback
#Callback stops training if maximum is reached in mean reward
callback_on_best = StopTrainingOnRewardThreshold(
reward_threshold=env2.calculate_threshold(), verbose=1)
# Callback safes the currently best model
eval_callback = EvalCallback(env4,
callback_on_new_best=callback_on_best,
verbose=1,
best_model_save_path='./ACKTR_Models/best/')
checkpoint_callback = CheckpointCallback(save_freq=1e4,
save_path='./model_checkpoints/')
# Uncomment, to train a new fresh model, otherwise a allready trained model will be trained
# If a frehs model is trained, it should be trained with binary reward (Config) first, to reduce multiple
# shots onto the same field.
def registerExtenderCallbacks(self, callbacks):
self.mCallBacks = callbacks
self.config = Config(callbacks)
self.loader = Loader(self.config)
self.dispatcher = Dispatcher(self.config, self.loader.getPlugins())
def run(self):
train_itr = 0
learn_count = 0
episode_frames = []
episode_count = 0
deleted_value = 0
deleted_age = 0
deleted_demo = 0
pre_score = 0
pre_train_itr = 0
#
lock = threading.Lock()
print(self.name)
count = 0
scores, e, replay_full_episode = [], 0, None
filename = ''
random.shuffle(self.episodeList)
epsidoe_list_count = 0
self.episode = self.episodeList[epsidoe_list_count]
self.i, self.f = goNextEpisode(self.i, self.f, self.episode)
episodeEnd = False
sample_log = openLog(Config.LEARNER_DATA_PATH + 'sampleexp/', '',
['step', 'value', 'age', 'demo', 'qvalue'])
replay_log = openLog(
Config.LEARNER_DATA_PATH + 'replaymemory/', '',
['step', 'root_priority', 'root_ts', 'root_demo', 'alpha', 'beta'])
delete_log = openLog(Config.ACTOR_DATA_PATH + 'deletedexp/', '',
['step', 'train_itr', 'value', 'age', 'demo'])
episode_log = openLog(Config.ACTOR_DATA_PATH + 'episodescore/', '',
['episode', 'score'])
actor_done, actor_score, actor_n_step_reward, actor_state = False, 0, None, self.env.reset(
)
human_done, human_score, human_n_step_reward, human_state = False, 0, None, np.zeros(
[83, 83, 3], dtype=np.float32)
episodeEnd = False
t_q_actor = deque(maxlen=Config.trajectory_n)
t_q_human = deque(maxlen=Config.trajectory_n)
episode_count = 0
train_itr = train_itr + 1
avg_actor_time_step = 0
act_itr = 0
while (learn_count < Config.LEARNER_TRAINING_STEP):
human_state = process_frame(human_state)
actor_state = process_frame(actor_state)
while actor_done is False and episodeEnd is False:
startTime = time.time()
if (train_itr % Config.ACTOR_HUMAN_COUNT != 0):
action = self.agent.egreedy_action(
actor_state) # e-greedy action for train
next_state, reward, actor_done, _ = self.env.step(action)
# env.render()
episode_frames.append(next_state)
next_state = process_frame(next_state)
# print(next_state)
actor_score += reward
reward = sign(reward) * math.log(
1 + abs(reward)) if not actor_done else sign(
-100) * math.log(1 + abs(-100))
reward_to_sub = 0. if len(
t_q_actor) < t_q_actor.maxlen else t_q_actor[0][
2] # record the earliest reward for the sub
t_q_actor.append([
actor_state, action, reward, next_state, actor_done,
0.0
])
if len(t_q_actor) == t_q_actor.maxlen:
if actor_n_step_reward is None: # only compute once when t_q first filled
actor_n_step_reward = sum([
t[2] * Config.GAMMA**i
for i, t in enumerate(t_q_actor)
])
else:
actor_n_step_reward = (
actor_n_step_reward -
reward_to_sub) / Config.GAMMA
actor_n_step_reward += reward * Config.GAMMA**(
Config.trajectory_n - 1)
t_q_actor[0].extend([
actor_n_step_reward, next_state, actor_done,
t_q_actor.maxlen, self.agent.time_step
]) # actual_n is max_len here
self.agent.perceive(
t_q_actor[0], self.agent.time_step
) # perceive when a transition is completed
# print(demo)
# print(t_q[0][3])
# print(self.learner.time_step)
actor_state = next_state
if (train_itr % Config.ACTOR_HUMAN_COUNT == 0):
startTime = time.time()
next_state, reward, human_done, action, episodeEnd = step(
self.i, self.f, self.episode)
self.i = self.i + 1
if (episodeEnd != True):
human_score += reward
reward = sign(reward) * math.log(
1 + abs(reward)) if not human_done else sign(
-100) * math.log(1 + abs(-100))
reward_to_sub = 0. if len(
t_q_human) < t_q_human.maxlen else t_q_human[0][
2] # record the earliest reward for the sub
t_q_human.append([
human_state, action, reward, next_state,
human_done, 1.0
])
# print(next_state)
if len(t_q_human) == t_q_human.maxlen:
if human_n_step_reward is None: # only compute once when t_q first filled
human_n_step_reward = sum([
t[2] * Config.GAMMA**i
for i, t in enumerate(t_q_human)
])
else:
human_n_step_reward = (
human_n_step_reward -
reward_to_sub) / Config.GAMMA
human_n_step_reward += reward * Config.GAMMA**(
Config.trajectory_n - 1)
t_q_human[0].extend([
human_n_step_reward, next_state, human_done,
t_q_human.maxlen, self.agent.time_step
]) # actual_n is max_len here
self.agent.perceive(
t_q_human[0], self.agent.time_step
) # perceive when a transition is completed
human_state = next_state
train_itr = train_itr + 1
if self.agent.replay_memory.full():
self.agent.train_Q_network(
update=False) # train along with generation
learn_count += 1
if (train_itr % Config.LEARNER_TRAINING_PART == 0):
self.agent.save_model()
sample_demo = float(self.agent.demo_num) / (
Config.LEARNER_TRAINING_PART * Config.BATCH_SIZE)
sample_value = math.pow(
self.agent.sum_abs_error /
(Config.LEARNER_TRAINING_PART * Config.BATCH_SIZE),
0.4)
sample_age = self.agent.sum_age / (
Config.LEARNER_TRAINING_PART * Config.BATCH_SIZE)
sample_q = self.agent.qvalue / (
Config.LEARNER_TRAINING_PART * Config.BATCH_SIZE)
print("learner_sample")
print(sample_value)
print(sample_age)
print(sample_demo)
print(sample_q)
sum_sample_q = 0
for i in range(6):
sum_sample_q += sample_q[i]
print(sum_sample_q)
self.agent.sum_abs_error = 0
self.agent.demo_num = 0
self.agent.sum_age = 0
self.agent.qvalue = 0
print("replay_memory")
print(self.agent.replay_memory.tree.total_p)
writeLog(Config.LEARNER_DATA_PATH + 'sampleexp/',
sample_log, [
str(train_itr),
str(sample_value),
str(sample_age),
str(sample_demo),
str(sum_sample_q)
])
writeLog(
Config.LEARNER_DATA_PATH + 'replaymemory/',
replay_log, [
str(train_itr),
str(self.agent.replay_memory.tree.total_p),
str(self.agent.replay_memory.tree.total_ts),
str(self.agent.replay_memory.tree.total_d),
str(self.agent.replay_memory.tree.alpha),
str(self.agent.replay_memory.tree.beta)
])
gc.collect()
replay_full_episode = replay_full_episode or e
if learn_count % Config().UPDATE_TARGET_NET == 0:
# print("actor_update_target"+str(train_itr))
self.agent.sess.run(self.agent.update_target_net)
if (train_itr % 100 == 0):
print("process time : " + str(time.time() - startTime) +
"/" +
str(self.agent.replay_memory.tree.data_pointer))
if (train_itr % Config.ACTOR_ACTING_PART == 0):
sum_value = self.agent.replay_memory.tree.avg_val / Config.ACTOR_ACTING_PART
sum_age = self.agent.replay_memory.tree.avg_time / Config.ACTOR_ACTING_PART
sum_demo = self.agent.replay_memory.tree.avg_demo / Config.ACTOR_ACTING_PART
print("actor_deleted")
print(sum_value)
print(sum_age)
print(sum_demo)
writeLog(Config.ACTOR_DATA_PATH + 'deletedexp/',
delete_log, [
str(count),
str(train_itr),
str(sum_value),
str(sum_age),
str(sum_demo)
])
self.agent.replay_memory.tree.avg_val = 0
self.agent.replay_memory.tree.avg_time = 0
self.agent.replay_memory.tree.avg_demo = 0
if actor_done:
# handle transitions left in t_q
t_q_actor.popleft() # first transition's n-step is already set
transitions = set_n_step(t_q_actor, Config.trajectory_n,
self.agent.time_step)
for t in transitions:
self.agent.perceive(t, self.agent.time_step)
if self.agent.replay_memory.full():
delta = actor_score - pre_score
actor_num = 1
sub_train_itr = learn_count - pre_train_itr
# print(sub_train_itr)
self.agent.replay_memory.update_alpha_and_beta(
delta, actor_num, sub_train_itr)
pre_train_itr = learn_count
pre_score = actor_score
# scores.append(score)
if replay_full_episode is not None:
print(
"episode: {} trained-episode: {} score: {} memory length: {} epsilon: {}"
.format(e, e - replay_full_episode, actor_score,
len(self.agent.replay_memory),
self.agent.epsilon))
writeLog(Config.ACTOR_DATA_PATH + 'episodescore/',
episode_log,
[str(episode_count),
str(actor_score)])
# 주기적으로 에피소드의 gif 를 저장하고, 모델 파라미터와 요약 통계량을 저장한다.
#if episode_count % Config.GIF_STEP == 0 and episode_count != 0 :
# time_per_step = 0.05
# images = np.array(episode_frames)
# make_gif(images, './frames/dqfd_image' + str(episode_count) + '.gif',
# duration=len(images) * time_per_step, true_image=True, salience=False)
actor_done, actor_score, actor_n_step_reward, actor_state = False, 0, None, self.env.reset(
)
t_q_actor = deque(maxlen=Config.trajectory_n)
episode_count = episode_count + 1
episode_frames = []
if (episodeEnd):
# handle transitions left in t_q
print("human : episode end")
t_q_human.popleft() # first transition's n-step is already set
transitions = set_n_step(t_q_human, Config.trajectory_n,
self.agent.time_step)
for t in transitions:
self.agent.perceive(t, self.agent.time_step)
if self.agent.replay_memory.full():
if train_itr % Config().UPDATE_TARGET_NET == 0:
#print("human_update_target")
self.agent.sess.run(self.agent.update_target_net)
epsidoe_list_count += 1
if (epsidoe_list_count == self.episodeList.__len__()):
random.shuffle(self.episodeList)
epsidoe_list_count = 0
self.episode = self.episodeList[epsidoe_list_count]
else:
self.episode = self.episodeList[epsidoe_list_count]
self.i, self.f = goNextEpisode(self.i, self.f, self.episode)
human_done, human_score, human_n_step_reward, human_state = False, 0, None, np.zeros(
[83, 83, 3], dtype=np.float32)
t_q_human = deque(maxlen=Config.trajectory_n)
episodeEnd = False
e += 1
print("actor end")
