def main():
args = make_args()
config = configparser.ConfigParser()
utils.load_config(config, args.config)
for cmd in args.modify:
utils.modify_config(config, cmd)
with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
logging.config.dictConfig(yaml.load(f))
model_dir = utils.get_model_dir(config)
path, step, epoch = utils.train.load_model(model_dir)
state_dict = torch.load(path, map_location=lambda storage, loc: storage)
mapper = [(inflection.underscore(name), member()) for name, member in inspect.getmembers(importlib.machinery.SourceFileLoader('', __file__).load_module()) if inspect.isclass(member)]
path = os.path.join(model_dir, os.path.basename(os.path.splitext(__file__)[0])) + '.xlsx'
with xlsxwriter.Workbook(path, {'strings_to_urls': False, 'nan_inf_to_errors': True}) as workbook:
worksheet = workbook.add_worksheet(args.worksheet)
for j, (key, m) in enumerate(mapper):
worksheet.write(0, j, key)
for i, (name, variable) in enumerate(state_dict.items()):
value = m(name, variable)
worksheet.write(1 + i, j, value)
if hasattr(m, 'format'):
m.format(workbook, worksheet, i, j)
worksheet.autofilter(0, 0, i, len(mapper) - 1)
worksheet.freeze_panes(1, 0)
logging.info(path)
def main():
args = make_args()
config = configparser.ConfigParser()
utils.load_config(config, args.config)
for cmd in args.modify:
utils.modify_config(config, cmd)
with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
logging.config.dictConfig(yaml.load(f))
model_dir = utils.get_model_dir(config)
height, width = tuple(map(int, config.get('image', 'size').split()))
resize = transform.parse_transform(config, config.get('transform', 'resize_test'))
transform_image = transform.get_transform(config, config.get('transform', 'image_test').split())
transform_tensor = transform.get_transform(config, config.get('transform', 'tensor').split())
# load image
image_bgr = cv2.imread('image.jpg')
image_resized = resize(image_bgr, height, width)
image = transform_image(image_resized)
tensor = transform_tensor(image).unsqueeze(0)
# Caffe2
init_net = caffe2_pb2.NetDef()
with open(os.path.join(model_dir, 'init_net.pb'), 'rb') as f:
init_net.ParseFromString(f.read())
predict_net = caffe2_pb2.NetDef()
with open(os.path.join(model_dir, 'predict_net.pb'), 'rb') as f:
predict_net.ParseFromString(f.read())
p = workspace.Predictor(init_net, predict_net)
results = p.run([tensor.numpy()])
logging.info(utils.abs_mean(results[0]))
logging.info(hashlib.md5(results[0].tostring()).hexdigest())
def main():
args = make_args()
config = configparser.ConfigParser()
utils.load_config(config, args.config)
for cmd in args.modify:
utils.modify_config(config, cmd)
with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
logging.config.dictConfig(yaml.load(f))
cache_dir = utils.get_cache_dir(config)
os.makedirs(cache_dir, exist_ok=True)
shutil.copyfile(os.path.expanduser(os.path.expandvars(config.get('cache', 'category'))), os.path.join(cache_dir, 'category'))
category = utils.get_category(config)
category_index = dict([(name, i) for i, name in enumerate(category)])
datasets = config.get('cache', 'datasets').split()
for phase in args.phase:
path = os.path.join(cache_dir, phase) + '.pkl'
logging.info('save cache file: ' + path)
data = []
for dataset in datasets:
logging.info('load %s dataset' % dataset)
module, func = dataset.rsplit('.', 1)
module = importlib.import_module(module)
func = getattr(module, func)
data += func(config, path, category_index)
if config.getboolean('cache', 'shuffle'):
random.shuffle(data)
with open(path, 'wb') as f:
pickle.dump(data, f)
logging.info('%s data are saved into %s' % (str(args.phase), cache_dir))
def init(self):
self = super(QSX, self).init()
self.injected_apps = []
self.groups = []
self.active_space = None
self.active_group = None
self.setup_observers()
hotkeys.init_default_qsx_hotkeys(self)
utils.load_config()
return self
def main():
args = make_args()
config = configparser.ConfigParser()
utils.load_config(config, args.config)
for cmd in args.modify:
utils.modify_config(config, cmd)
with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
logging.config.dictConfig(yaml.load(f))
model_dir = utils.get_model_dir(config)
init_net = onnx_caffe2.helper.load_caffe2_net(os.path.join(model_dir, 'init_net.pb'))
predict_net = onnx_caffe2.helper.load_caffe2_net(os.path.join(model_dir, 'predict_net.pb'))
benchmark = onnx_caffe2.helper.benchmark_caffe2_model(init_net, predict_net)
logging.info('benchmark=%f(milliseconds)' % benchmark)
def main():
args = make_args()
config = configparser.ConfigParser()
utils.load_config(config, args.config)
for cmd in args.modify:
utils.modify_config(config, cmd)
with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
logging.config.dictConfig(yaml.load(f))
model_dir = utils.get_model_dir(config)
model = onnx.load(model_dir + '.onnx')
onnx.checker.check_model(model)
init_net, predict_net = onnx_caffe2.backend.Caffe2Backend.onnx_graph_to_caffe2_net(model.graph, device='CPU')
onnx_caffe2.helper.save_caffe2_net(init_net, os.path.join(model_dir, 'init_net.pb'))
onnx_caffe2.helper.save_caffe2_net(predict_net, os.path.join(model_dir, 'predict_net.pb'), output_txt=True)
logging.info(model_dir)
def check_site_id(arg=None):
import db
config = utils.load_config()
data = db.Connection(host='127.0.0.1',database='lamp',user=config['lampuser'],password=config['lamppass'])
sid = -1
sites = data.query('SELECT id, domain, site_root, alias FROM lamp.lamp__sites ORDER BY id')
if arg and arg[0].isdigit():
sid = arg[0]
while True:
if sid == -1:
print 'ID\tdomain\t\tpath'
for s in sites:
print '%s\t%s \t%s' % (s.id, s.domain, s.site_root)
sid = raw_input('[ID]:')
for s in sites:
if str(s.id) == sid:
return s
sid = -1
def check_mysql_id(arg):
import db
config = utils.load_config()
data = db.Connection(host='127.0.0.1',database='lamp',user=config['lampuser'],password=config['lamppass'])
mid = -1
# get all mysql user
mysqls = data.query('SELECT m.id, m.site_id, m.login_name, s.domain FROM lamp__mysql AS m, lamp__sites AS s WHERE m.site_id = s.id ORDER BY m.id')
if arg and arg[0].isdigit():
mid = arg[0]
while True:
if mid == -1:
print '\nID\tuser\t\tsite'
for m in mysqls:
print '%s \t%s \t%s' % (m.id, m.login_name, m.domain)
mid = raw_input('Please input MySQL id:')
for m in mysqls:
if str(m.id) == mid:
return [mid, m.login_name]
# no found in this loop
mid = -1
def reload_plugins(*args, **kwargs):
"""Module function that'll reload all of the plugins"""
config = utils.load_config("Pyhole", kwargs.get("conf_file"))
# When the modules are reloaded, the meta class will append
# all of the classes again, so we need to make sure this is empty
Plugin._plugin_classes = []
_reset_variables()
# Now reload all of the plugins
plugins_to_reload = []
plugindir = "pyhole.plugins"
# Reload existing plugins
for mod, val in sys.modules.items():
if plugindir in mod and val and mod != plugindir:
mod_file = val.__file__
if not os.path.isfile(mod_file):
continue
for p in config.get("plugins", type="list"):
if plugindir + "." + p == mod:
plugins_to_reload.append(mod)
for p in plugins_to_reload:
try:
reload(sys.modules[p])
except Exception, e:
# Catch all because this could be many things
kwargs.get("irc").log.error(e)
pass
def check_ftp_id(arg=None):
import db
config = utils.load_config()
data = db.Connection(host='127.0.0.1',database='lamp',user=config['lampuser'],password=config['lamppass'])
fid = -1
ftps = data.query('SELECT f.id, f.site_id, f.username, f.local_root, s.domain FROM lamp__ftp AS f, lamp__sites AS s WHERE f.site_id = s.id ORDER BY f.id')
if arg and arg[0].isdigit():
fid = arg[0]
while True:
if fid == -1:
print '\nID\tuser\t\tpath\t\tsite'
for f in ftps:
print '%s \t%s \t%s \t%s' % (f.id, f.username, f.local_root, f.domain)
fid = raw_input('Please input FTP id:')
for f in ftps:
if str(f.id) == fid:
return f
fid = -1
def delete_mysql(arg=None):
config = utils.load_config()
m = check_mysql_id(arg)
logging.debug('get mysql info m: %s', m)
if not utils.get_yseorno('Do you really want to delete mysql user: %s and drop databases that have the same names as the users?' % m[1]):
sys.exit()
import db
config = json.load(open('/etc/lamp/config'))
if utils.get_yseorno('Do you want to backup database?'):
bk_name = backup_database(m[1], config)
print 'Database backup to %s' % bk_name
data = db.Connection(host='127.0.0.1',database='lamp',user=config['lampuser'],password=config['lamppass'])
try:
r_mysql = data.get('SELECT user, host FROM mysql.user WHERE user = \"%s\"' % m[1])
logging.debug('del mysql %s, %s', m, r_mysql)
if r_mysql:
data.execute('DROP USER \"%s\"@\"%s\"' % (r_mysql.user, r_mysql.host))
data.execute('DROP DATABASE %s' % m[1])
data.execute('DELETE FROM lamp__mysql WHERE login_name = \"%s\"' % r_mysql.user)
else:
logging.info('Nothing to delete..')
except Exception,e:
logging.info('delete mysql and database has some errors : %s', str(e))
def verify(self, config_path, name, type):
try:
share = load_config(config_path, '{0}-{1}'.format(type, name))
except FileNotFoundError:
raise VerifyException(
errno.ENOENT,
'There is no share {0} of type {1} at {2} to be imported.'.format(name, type, config_path)
)
except ValueError:
raise VerifyException(
errno.EINVAL,
'Cannot read configuration file. File is not a valid JSON file'
)
if share['type'] != type:
raise VerifyException(
errno.EINVAL,
'Share type {0} does not match configuration file entry type {1}'.format(type, share['type'])
)
if not self.dispatcher.call_sync('share.supported_types').get(share['type']):
raise VerifyException(errno.ENXIO, 'Unknown sharing type {0}'.format(share['type']))
if self.datastore.exists(
'shares',
('type', '=', share['type']),
('name', '=', share['name'])
):
raise VerifyException(errno.EEXIST, 'Share {0} of type {1} already exists'.format(
share['name'],
share['type']
))
return ['system']
def run(self, name, volume):
try:
container = load_config(
self.dispatcher.call_sync(
'volume.resolve_path',
volume,
os.path.join('vm', name)
),
'vm-{0}'.format(name)
)
except FileNotFoundError:
raise TaskException(errno.ENOENT, 'There is no {0} on {1} volume to be imported.'. format(name, volume))
except ValueError:
raise VerifyException(
errno.EINVAL,
'Cannot read configuration file. File is not a valid JSON file'
)
id = self.datastore.insert('containers', container)
self.dispatcher.dispatch_event('container.changed', {
'operation': 'create',
'ids': [id]
})
return id
def delete_site(domain):
import db
config = utils.load_config()
data = db.Connection(host='127.0.0.1',database='lamp',user=config['lampuser'],password=config['lamppass'])
if domain.isdigit():
site = data.get('SELECT id, domain, site_root FROM lamp.lamp__sites WHERE id = %s' % domain)
else:
site = data.get('SELECT id, domain, site_root FROM lamp.lamp__sites WHERE domain = "%s"' % domain)
if not site:
logging.info('domain %s don\'t exists, Please pick one:', domain)
site = check_site_id()
logging.debug('delete site domain: %s', domain)
# don't delete default site
if site.id == 1:
logging.warning('You can\'t delete default site')
sys.exit(-1)
try:
data.execute('DELETE FROM lamp.lamp__sites WHERE domain = "%s"' % domain)
except Exception,e:
logging.info('delete site to mysql has some errors : %s', str(e))
site = -0
def file_tree(conf='basic', depcfg='data/deps1.cfg', output_files=('Output.js',)):
#@@ add a dep->out map option
tempdir, libdir = testutils.setup_temp_dir()
merger = testutils.load_config(conf, libdir)
handles = testutils.setup_dir(merger, prefix=libdir)
depmap = deps.DepMap.from_resource(depcfg)
files = [x for x in testutils.inject_deps(handles, libdir, depmap)]
set_faux_files(merger, libdir, *output_files)
return merger
def __init__(self, server='', player='p1', idx=0):
pygame.init()
pygame.font.init()
self.em = EventManager()
self.em.add_event(Event("game-event"))
self.config = load_config(CONFIG_FILE)
self.menu_options = self.config['MENU']
self.idx = idx
self.mode = "GAME"
self.server = server
self.player_name = player
self.remote_game = None
if self.server:
self.remote_game = RemoteGame(self)
self.window = pygame.display.set_mode((self.config['SCREENHEIGHT'], self.config['SCREENWIDTH']))
pygame.display.set_caption(self.config['TITLE'])
self.screen = pygame.display.get_surface()
self.clock = pygame.time.Clock()
self.chat_surface = pygame.Surface((self.screen.get_width(), 50))
self.chat_surface.fill(pygame.Color(255, 255, 255))
self.chat_surface.set_alpha(130)
self.menu_surface = pygame.Surface((self.screen.get_width(), self.screen.get_height() // 2))
self.menu_surface.fill(pygame.Color(255, 255, 255))
self.menu_surface.set_alpha(130)
self.text = ""
self.font = pygame.font.SysFont("Sans", 16)
self.map = Map(1, 1)
self.map.load_from_image(self.config['MAP'])
self.map.scroll = [0, 14]
self.guy1 = Guy(self.idx, self)
self.guy1.set_name(self.player_name)
self.guy1.set_pos(17, 0)
self.guy2 = RemoteGuy(1, self)
self.guy2.set_name("Cat girl")
self.guy2.movement = "circular"
self.guy2.set_pos(15, 0)
tree1 = Tree(self, 0)
tree2 = Tree(self, 1)
tree3 = Tree(self, 1)
tree4 = Tree(self, 2)
tree2.set_pos(20, 0)
tree1.set_pos(20, 1)
tree3.set_pos(19, 2)
tree4.set_pos(18, 3)
self.events = {}
def __init__(self, request):
super(Metadata_View, self).__init__(request)
self.check_login()
self.templatePath = 'templates/metadata.pt'
self.config = load_config(request)
self.metadata = self.config['metadata']
self.featured_links = []
self.workspaces = \
[(i['href'], i['title']) for i in self.session['collections']]
self.role_mappings = {'authors': 'dcterms:creator',
'maintainers': 'oerdc:maintainer',
'copyright': 'dcterms:rightsHolder',
'editors': 'oerdc:editor',
'translators': 'oerdc:translator'}
self.subjects = ["Arts",
"Business",
"Humanities",
"Mathematics and Statistics",
"Science and Technology",
"Social Sciences",
]
# The roles fields are comma-separated strings. This makes the javascript
# easier on the client side, and is easy to parse. The fields are hidden,
# and the values will be user ids, which should not have commas in them.
self.field_list = [
['authors', 'authors', {'type': 'hidden'}],
['maintainers', 'maintainers', {'type': 'hidden'}],
['copyright', 'copyright', {'type': 'hidden'}],
['editors', 'editors', {'type': 'hidden'}],
['translators', 'translators', {'type': 'hidden'}],
['title', 'Title', {'type': 'text'}],
['summary', 'Summary', {'type': 'textarea'}],
['keywords', 'Keywords (One per line)', {'type': 'textarea'}],
['subject', 'Subject', {'type': 'checkbox',
'values': self.subjects}],
['language', 'Language', {'type': 'select',
'values': languages,
'selected_value': 'en'}],
['google_code', 'Google Analytics Code', {'type': 'text'}],
['workspace', 'Workspace', {'type': 'select',
'values': self.workspaces}],
]
self.remember_fields = [field[0] for field in self.field_list[5:]]
# Get remembered fields from the session
self.defaults = {}
# Get remembered title from the session
if 'title' in self.session:
self.defaults['title'] = self.session['title']
self.config['metadata']['title'] = self.session['title']
def update_site_alias(site_id, alias):
import db
config = utils.load_config()
data = db.Connection(host='127.0.0.1',database='lamp',user=config['lampuser'],password=config['lamppass'])
alias = json.dumps(alias)
try:
data.execute('UPDATE lamp__sites SET alias=\'%s\' WHERE id=%s' % (alias, site_id))
except Exception,e:
logging.info('update site alias has some errors: %s', str(e))
return False
def __init__(self, parent=None, config_file="src/demos/temporal/config.json"):
super(HinselmannResults, self).__init__(parent)
self.config = utils.load_config(config_file)
self.width = 0
self.height = 0
self.width, self.height = wx.DisplaySize()
self.parent = parent
self.setLayout()
self.addTooltips()
self.bindControls()
def run(self, config_path, name, type):
share = load_config(config_path, '{0}-{1}'.format(type, name))
ids = self.join_subtasks(self.run_subtask('share.{0}.import'.format(share['type']), share))
self.dispatcher.dispatch_event('share.changed', {
'operation': 'create',
'ids': ids
})
return ids[0]
def __init__(self, parent=None,
config_file="src/demos/temporal/config.json"):
super(DiagnosisPanel, self).__init__(parent,)
self.config = utils.load_config(config_file)
self.width = 0
self.height = 0
self.width, self.height = wx.DisplaySize()
self.setLayout()
self.addTooltips()
self.bindControls()
def get_ftps_bysite(site_id):
import db
config = utils.load_config()
data = db.Connection(host='127.0.0.1',database='lamp',user=config['lampuser'],password=config['lamppass'])
ftps = data.query('SELECT f.id FROM lamp__ftp AS f WHERE f.site_id = %s' % site_id)
logging.debug('delete ftps by site_id: %s, ftps: %s', site_id, ftps)
del data
return ftps
def reset_mysql_pass(arg=None):
m = check_mysql_id(arg)
import db
config = utils.load_config()
data = db.Connection(host='127.0.0.1',database='lamp',user=config['lampuser'],password=config['lamppass'])
user_pass = utils.gen_random_str()
try:
data.execute('UPDATE mysql.user SET password=PASSWORD(\'%s\') WHERE user=\'%s\'' % (user_pass, m[1]))
data.execute('FLUSH PRIVILEGES')
except Exception,e:
logging.info('delete mysql and database has some errors : %s', str(e))
sys.exit(1)
def cnxlogin_view(request):
check_login(request)
config = load_config(request)
login_url = config['login_url']
templatePath = 'templates/cnxlogin.pt'
response = {
'username': request.session['login'].username,
'password': request.session['login'].password,
'login_url': login_url,
}
return render_to_response(templatePath, response, request=request)
def send_text():
"""Sends the txt message from data passed through POST."""
if request.headers['Content-Type'] == 'application/json':
#converts json to python dict
data = request.json
#get list of carriers from carriers.json
config = load_config()
carriers = load_carriers()
#authenticate request
if data['api_key'] == config['api_key']:
if data['carrier'] in carriers:
#prepare the message
carrier_choice = data['carrier']
carrier = carriers[carrier_choice]
number = data['number']
msg = data['msg']
to = "{0}{1}".format(number, carrier)
sender = config['from']
#sends the actual message
mail = smtplib.SMTP(config['smtp_address'])
mail.starttls()
mail.login(config['username'], config['password'])
mail.sendmail(sender, to, msg)
mail.quit()
#prepare the json response.
log = "Message: '{0}' was sent succesfuly sent to '{1}'.".format(msg, to)
logging(log)
resp = {"response" : log}
response = Response(json.dumps(resp), status=200, mimetype='application/json')
return response
#if the carrier is not supported or found in the carriers list.
else:
log = "Carrier not supported."
#log to web2txt.log file
logging(log)
resp = {"response" : log}
response = Response(json.dumps(resp), status=404, mimetype='application/json')
return response
#if the content type is not json
else:
log = "Wrong request content-type. API only support JSON"
#log to web2txt.log file
logging(log)
resp = {"response" : log}
response = Response(json.dumps(resp), status=415, mimetype='application/json')
return response
def load_plugins(*args, **kwargs):
"""Module function that loads plugins from a particular directory"""
config = utils.load_config("Pyhole", kwargs.get("conf_file"))
plugin_names = config.get("plugins", type="list")
for p in plugin_names:
load_user_plugin(p, *args, **kwargs)
try:
__import__("pyhole.plugins", globals(), locals(), [p])
except Exception, e:
# Catch all because this could be many things
kwargs.get("irc").log.error(e)
pass
def deploy_dir(path, kwargs):
with utils.directory(path):
config = utils.load_config()
config['config'].update(kwargs)
if 'FunctionName' not in config['config']:
clip.exit('You must provide a function name', err=True)
# Remove ignore paths
for e in config['ignore'] + ['.git/', '.gitignore']:
utils.delete_resource(e)
# Run install command
if 'install' in config:
utils.shell(config['install'])
# Zip up directory
utils.make_zip(config['config']['FunctionName'])
# Upload!
params = config['config']
upload(params['FunctionName'], params)
def create_site(domain, site_root):
import db
config = utils.load_config()
data = db.Connection(host='127.0.0.1',database='lamp',user=config['lampuser'],password=config['lamppass'])
site = data.get('select domain, site_root from lamp.lamp__sites where domain = "%s"' % domain)
if site:
logging.info('domain %s already exists, exiting...', domain)
sys.exit(1)
logging.debug('insert site domain: %s, site_root: %s', domain, site_root)
try:
site = data.insert('lamp__sites',domain=domain, site_root=site_root)
except Exception,e:
logging.info('insert site to mysql has some errors : %s', str(e))
site = -0
def get_ftps(arg=None):
import db
config = utils.load_config()
data = db.Connection(host='127.0.0.1',database='lamp',user=config['lampuser'],password=config['lamppass'])
if not arg:
ftps = data.query('SELECT f.id, f.site_id, f.username, f.local_root, s.domain FROM lamp__ftp AS f, lamp__sites AS s WHERE f.site_id = s.id ORDER BY f.id')
elif arg[0].isdigit():
ftps = data.query('SELECT f.id, f.site_id, f.username, f.local_root, s.domain FROM lamp__ftp AS f, lamp__sites AS s WHERE f.site_id = s.id AND f.id = %s ORDER BY f.id' % arg[0])
else:
ftps = data.query('SELECT f.id, f.site_id, f.username, f.local_root, s.domain FROM lamp__ftp AS f, lamp__sites AS s WHERE f.site_id = s.id AND f.username LIKE \"%%{0}%%\" ORDER BY f.id'.format(arg[0]))
print 'FTP(%i):' % len(ftps)
if ftps:
for f in ftps:
print '-' * 50
print '\nID\tuser\t\tpath\t\tsite'
print '%s \t%s \t%s \t%s' % (f.id, f.username, f.local_root, f.domain)
def do_startup(self):
Gtk.Application.do_startup(self)
self.config = utils.load_config()
add_account_action = Gio.SimpleAction.new("add_account", None)
add_account_action.connect("activate", self.add_account_callback)
self.add_action(add_account_action)
new_action = Gio.SimpleAction.new("new", None)
new_action.connect("activate", self.on_new)
self.add_action(new_action)
preview_action = Gio.SimpleAction.new("preview", None)
preview_action.connect("activate", self.on_preview)
self.add_action(preview_action)
open_action = Gio.SimpleAction.new("open_draft", None)
open_action.connect("activate", self.on_open_draft)
self.add_action(open_action)
save_action = Gio.SimpleAction.new("save_draft", None)
save_action.connect("activate", self.on_save_draft)
self.add_action(save_action)
remove_current_blog_action = Gio.SimpleAction.new("remove_current_blog", None)
remove_current_blog_action.connect("activate", self.on_remove_current_blog)
self.add_action(remove_current_blog_action)
quit_action = Gio.SimpleAction.new("quit", None)
quit_action.connect("activate", self.on_quit)
self.add_action(quit_action)
for item in self.config["blogs"]:
self.create_select_blog_action(item["id"])
# Insert actions
insert_image_action = Gio.SimpleAction.new("insert_image", None)
insert_image_action.connect("activate", self.on_insert_image)
self.add_action(insert_image_action)
insert_code_action = Gio.SimpleAction.new("insert_code", None)
insert_code_action.connect("activate", self.on_insert_code)
self.add_action(insert_code_action)
def get_hash(config_file, token):
prompt_warning('File %s will be changed.' % config_file)
config = load_config(config_file)
github = Github(config['player'], token)
return start_get_hash(config, github, config_file)
def __init__(self, file_=None, **kwargs):
self.default = utils.load_config(self, file_)
utils.set_mosdirs(self)
self.config_file_name = file_
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
# update_tag_scheme(train_sentences, FLAGS.tag_schema)
# update_tag_scheme(test_sentences, FLAGS.tag_schema)
# update_tag_scheme(dev_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences, dev_sentences,
test_sentences)
#with open('maps.txt','w',encoding='utf8') as f1:
#f1.writelines(str(char_to_id)+" "+id_to_char+" "+str(tag_to_id)+" "+id_to_tag+'\n')
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 10)
test_manager = BatchManager(test_data, 10)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
# tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
with tf.device("/cpu:0"):
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
# best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if i % 7 == 0:
save_model(sess, model, FLAGS.ckpt_path, logger)
# Custom imports
sys.path.append('autoencoders/')
import EviAE
import ConvAE, SAE
##############################################################################
# Training mechanism of EviTRAM
##############################################################################
########## Initialization of experiments properties, datasets etc. ########
# Load configuration files
try:
# Main autoencoder config file
cp = utils.load_config(sys.argv[1])
except:
print 'Help: ./train.py <path to main autoencoder ini file> <run number>'
exit()
# Trying to reduce stochastic behaviours
SEED = cp.getint('Experiment', 'SEED')
tf.set_random_seed(SEED)
np.random.seed(SEED)
# Load dataset
inp_path = cp.get('Experiment', 'DATAINPUTPATH')
if inp_path == '':
dataset = utils.load_mnist(
val_size=cp.getint('Experiment', 'VALIDATIONSIZE'))
else:
frame_length {int} -- the number of samples in each analysis window
overlap {float} -- in [0, 1) the fraction of overlap for each window
returns: np.array with dimensions (nb_mfcc,t,n_windows)
"""
mfcc = generate_MFCC(array, n_mfcc, frame_length, overlap, sampling_rate,
n_windows)
mfcc_formatted = np.array(np.split(mfcc, 30, axis=1))
return mfcc_formatted
if __name__ == "__main__":
params = load_params()
config = load_config()
data = read_in_data(params["sampling_rate"],
sample_data=config["using_sample_data"])
sample_array = data['test'][5][1]
sample_array = np.array(sample_array.tolist() + list(range(10000)))
x = generate_MFCC(sample_array,
n_mfcc=params["MFCC"]["n_mfcc"],
frame_length=params["MFCC"]["frame_length_in_s"],
overlap=params["MFCC"]["overlap"],
sampling_rate=params["sampling_rate"],
n_windows=params["MFCC"]["n_windows"])
#Output format should be (40, 1500)
print(x.shape)
def train_new():
train_sent = load_sentences(FLAGS.filepath)
update_tag_scheme(train_sent, FLAGS.tag_schema)
if not os.path.isfile(FLAGS.map_file):
_c, char_to_id, id_to_char = char_mapping(train_sent, FLAGS.lower)
print("random embedding")
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sent)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# 数据准备，划分验证集和训练集
np.random.seed(10)
train_sent_ = np.array(train_sent)
shuffle_indices = np.random.permutation(np.arange(len(train_sent)))
sent_shuffled = train_sent_[shuffle_indices]
dev_sample_index = -1 * int(FLAGS.dev_percentage * float(len(train_sent)))
train_sent_new, dev_sent = sent_shuffled[:dev_sample_index], sent_shuffled[
dev_sample_index:]
train_data = prepare_dataset(train_sent_new, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sent, char_to_id, tag_to_id, FLAGS.lower)
print("%i / %i sentences in train." % (len(train_data), len(dev_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = FLAGS.log_file
logger = get_logger(log_path)
print_config(config, logger)
# 根据需求，设置动态使用GPU资源
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
fig = plt.figure()
ax = fig.add_subplot(211)
ax2 = fig.add_subplot(212)
plt.grid(True)
plt.ion()
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(FLAGS.max_epoch):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % 20 == 0:
ax.scatter(step, np.mean(loss), c='b', marker='.')
plt.pause(0.001)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best, f1 = evaluate(sess, model, "dev", dev_manager, id_to_tag,
logger)
ax2.scatter(i + 1, f1, c='b', marker='.')
plt.pause(0.001)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger, "best")
from model import teacher_model
# download dataset with flowers 5 different classes
dataset_url = "https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz"
data_dir = tf.keras.utils.get_file('flower_photos',
origin=dataset_url,
untar=True)
data_dir = pathlib.Path(data_dir)
# check if it was well downloaded
image_count = len(list(data_dir.glob('*/*.jpg')))
if image_count:
print("Data loaded correctly")
# load config
config = load_config(
str(pathlib.Path(__file__).parent.absolute()) + "/config.yaml")
if config:
print("Config loaded correctly")
# get values from config
val_spl = config["teacher"]["train"]["val_split"]
im_h, im_w = config["teacher"]["train"]["im_h"], config["teacher"]["train"][
"im_w"]
b_s = config["teacher"]["train"]["b_s"]
n_cl = config["teacher"]["train"]["n_classes"]
ep = config["teacher"]["train"]["epochs"]
# divide data on train and validation
# load train part
train_ds = tf.keras.preprocessing.image_dataset_from_directory(
data_dir,
def main():
args = parser.parse_args()
config = load_config(os.path.join(args.config_dir, "config.pkl"))
is_monocular = False
# add missing param to multiwarp stack run
# if "multiwarp-5" in config.name:
# if "/stack" in config.name:
# config.cameras_stacked="input"
# multiwarp-5
if "multiwarp-5" in config.name or "multiwarp-outer" in config.name or "multiwarp-all" in config.name:
mode = "multiwarp-5"
elif "singlewarp" in config.name:
mode = "singlewarp"
else:
raise ValueError("unknown mode")
if "/epi" in config.name:
enc = "epi"
elif "/stack" in config.name:
enc = "stack"
elif "/focalstack-17-5" in config.name:
enc = "focalstack-17-5"
elif "/focalstack-17-9" in config.name:
enc = "focalstack-17-9"
elif "/monocular" in config.name:
enc = "monocular"
is_monocular = True
else:
raise ValueError("unknown encoding")
if args.no_pose:
print(
"Considering zero ground truth pose. Only depth is predicted correctly in this case."
)
## data on which the pipeline has to be run
# config.data = "/home/dtejaswi/Documents/Projects/student_projects/joseph_daniel/data/module-1-1/module1-1-png"
# config.data = "/home/dtejaswi/Desktop/joseph_daniel/extras/png/A/60"
config.data = args.data_dir
## directory for storing output
# config.save_path = os.path.join("/home/dtejaswi/Desktop/joseph_daniel/ral/", mode, enc)
# output_dir = os.path.join(config.save_path, "results", args.seq)
output_dir = os.path.join(args.config_dir, "results",
args.suffix + args.seq)
if args.use_checkpoint_at is not None:
config.dispnet = os.path.join(
args.config_dir,
"dispnet_" + args.use_checkpoint_at + "_checkpoint.pth.tar")
output_dir = output_dir + "_epoch_" + args.use_checkpoint_at
else:
# load configuration from checkpoints
if args.use_latest_not_best:
config.dispnet = os.path.join(args.config_dir,
"dispnet_checkpoint.pth.tar")
output_dir = output_dir + "-latest"
else:
config.dispnet = os.path.join(args.config_dir,
"dispnet_best.pth.tar")
os.makedirs(output_dir)
os.makedirs(output_dir + "/disp")
# define transformations
transform = custom_transforms.Compose([
custom_transforms.ArrayToTensor(),
custom_transforms.Normalize(mean=0.5, std=0.5)
])
# Load validation dataset loaders
if config.lfformat == 'focalstack':
dataset = get_validation_focal_stack_loader(config, args.seq,
transform, args.no_pose)
print("Loading images as focalstack")
elif config.lfformat == 'stack':
dataset = get_validation_stacked_lf_loader(config, args.seq, transform,
args.no_pose, is_monocular)
print("Loading images as stack")
elif config.lfformat == 'epi':
dataset = get_validation_epi_loader(config, args.seq, transform,
args.no_pose)
print("Loading images as tiled EPIs")
else:
raise TypeError(
"Unknown light field image format. Should be either focalstack, stack or epi"
)
output_channels = len(config.cameras)
if config.lfformat == 'epi':
if config.cameras_epi == "vertical":
disp_encoder = EpiEncoder('vertical', config.tilesize).to(device)
dispnet_input_channels = 16 + len(
config.cameras
) # 16 is the number of output channels of the encoder
elif config.cameras_epi == "horizontal":
disp_encoder = EpiEncoder('horizontal', config.tilesize).to(device)
dispnet_input_channels = 16 + len(
config.cameras
) # 16 is the number of output channels of the encoder
elif config.cameras_epi == "full":
disp_encoder = EpiEncoder('full', config.tilesize).to(device)
if config.without_disp_stack:
dispnet_input_channels = 32 # 16 is the number of output channels of each encoder
else:
dispnet_input_channels = 32 + 5 # 16 is the number of output channels of each encoder, 5 from stack
else:
raise ValueError("Incorrect cameras epi format")
print("Initialised disp and pose encoders")
else:
disp_encoder = None
dispnet_input_channels = dataset[0]['tgt_lf_formatted'].shape[0]
print(
f"[DispNet] Using {dispnet_input_channels} input channels, {output_channels} output channels"
)
# Load disp net
disp_net = DispNetS(in_channels=dispnet_input_channels,
out_channels=output_channels,
encoder=disp_encoder).to(device)
weights = torch.load(config.dispnet)
disp_net.load_state_dict(weights['state_dict'])
disp_net.eval()
print("Loaded only dispnet")
# prediction
for i, validData in enumerate(dataset):
print("{:03d}/{:03d}".format(i + 1, len(dataset)), end="\r")
tgt = validData['tgt_lf'].unsqueeze(0).to(device)
ref = [r.unsqueeze(0).to(device) for r in validData['ref_lfs']]
if config.lfformat == "epi" and config.cameras_epi == "full":
tgt_lf_formatted_h = validData['tgt_lf_formatted_h'].unsqueeze(
0).to(device)
tgt_lf_formatted_v = validData['tgt_lf_formatted_v'].unsqueeze(
0).to(device)
tgt_stack = validData['tgt_stack'].unsqueeze(0).to(device)
# Encode the epi images further
if config.without_disp_stack:
# Stacked images should not be concatenated with the encoded EPI images
tgt_encoded_d = disp_net.encode(tgt_lf_formatted_v, None,
tgt_lf_formatted_h)
else:
# Stacked images should be concatenated with the encoded EPI images
tgt_encoded_d = disp_net.encode(tgt_lf_formatted_v, tgt_stack,
tgt_lf_formatted_h)
else:
tgt_formatted = validData['tgt_lf_formatted'].unsqueeze(0).to(
device)
if disp_net.has_encoder():
# This will only be called for epi with horizontal or vertical only encoding
if config.without_disp_stack:
# Stacked images should not be concatenated with the encoded EPI images
tgt_encoded_d = disp_net.encode(tgt_formatted, None)
else:
# Stacked images should be concatenated with the encoded EPI images
# NOTE: Here we stack all 17 images, not 5. Here the images missing from the encoding,
# are covered in the stack. We are not using this case in the paper at all.
tgt_encoded_d = disp_net.encode(tgt_formatted, tgt)
else:
# This will be called for focal stack and stack, where there is no encoding
tgt_encoded_d = tgt_formatted
disp = disp_net(tgt_encoded_d)
# print(output.shape)
# print(pose.shape)
# print(tgt.shape)
# print(ref[0].shape)
outfile = os.path.join(output_dir, "{:06d}.png".format(i))
plt.imsave(outfile, tgt.cpu().numpy()[0, 0, :, :], cmap='gray')
outfile = os.path.join(output_dir, "disp/{:06d}.png".format(i))
plt.imsave(outfile, disp.cpu().numpy()[0, 0, :, :])
outfile = os.path.join(output_dir, "disp/{:06d}.npy".format(i))
np.save(outfile, disp.cpu().numpy()[0, 0, :, :])
print("\nok")
parser.add_argument("--train", action="store_true")
parser.add_argument("--map_size", type=int, default=125)
parser.add_argument("--greedy", action="store_true")
parser.add_argument("--name", type=str, default="trusty-battle")
parser.add_argument("--eval", action="store_true")
parser.add_argument('--alg',
default='dqn',
choices=['dqn', 'drqn', 'a2c', 'ppo'])
parser.add_argument("--savedir", default='data/battle_model_3_players')
args = parser.parse_args()
# set logger
buffer.init_logger(args.name)
# init the game
env = magent.GridWorld(utils.load_config(args.map_size))
# env.set_render_dir("build/render")
# two groups of agents
handles = env.get_handles()
# sample eval observation set
eval_obs = [None, None, None]
if args.eval:
print("sample eval set...")
env.reset()
utils.generate_map(env, args.map_size, handles)
for i in range(len(handles)):
eval_obs[i] = buffer.sample_observation(env, handles, 2048, 500)
# load models
use_mpi = bool(sys.argv[2])
print("use_mpi:", use_mpi)
else:
print("Usage: analyse_sims.py config_file [use_mpi]")
sys.exit(1)
# Begin MPI
if use_mpi:
comm = MPI.COMM_WORLD
myid = comm.Get_rank()
else:
comm = None
myid = 0
# Load config file
cfg = utils.load_config(config_file, default_cfg())
cfg_spec = cfg['sim_spec']
cfg_diffuse = cfg['sim_diffuse']
cfg_out = cfg['sim_output']
cfg_beam = cfg['sim_beam']
cfg_gain = cfg['sim_gain']
cfg_noise = cfg['sim_noise']
# Construct array layout to simulate
if cfg_spec['use_legacy_array']:
# This is the deprecated legacy function
ants = utils.build_array()
else:
ants = utils.build_hex_array(hex_spec=cfg_spec['hex_spec'],
ants_per_row=cfg_spec['hex_ants_per_row'],
d=cfg_spec['hex_ant_sep'])
if self.return_sequence:
return output
else:
return output[:, -1]
if __name__ == '__main__':
from utils import load_config
from loss import DiceLoss
from tqdm import tqdm
import torch.optim as optim
config_file = 'config.yaml'
config = load_config(config_file)
input_modalites = int(config['PARAMETERS']['input_modalites'])
output_channels = int(config['PARAMETERS']['output_channels'])
base_channel = 4
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
def count_params(model):
''' print number of trainable parameters and its size of the model'''
num_of_param = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print('Model {} : params number {}, params size: {:4f}M'.format(
model._get_name(), num_of_param, num_of_param * 4 / 1000 / 1000))
model = CenterLSTM(input_modalites,
output_channels,
with closing(HTTPConnection(host, port)) as con:
con.request('DELETE', filepath + '?lock_id=%s' % lock_id)
status = con.getresponse().status
if status != 200:
raise DFSIOError('Error (%d) while deleting %s.' %
(status, filepath))
def rename(filepath, newfilepath):
"""Rename filepath to newfilepath."""
with open(filepath) as f:
with open(newfilepath, 'w') as nf:
nf.write(f.read())
unlink(filepath, f.lock_id)
open = File
_config = {
'nameserver': None,
'lockserver': None,
'max_size': 1024**2,
} # default
File._cache = {}
utils.load_config(_config, 'client.dfs.json')
json.dump(meta_data, json_file)
# Creating .mcpr zipfile based on timestamp
print('Creating .mcpr file...')
date = datetime.datetime.today().strftime('SARC_%Y%m%d_%H_%S')
zipf = zipfile.ZipFile(date + '.mcpr', 'w', zipfile.ZIP_DEFLATED)
zipf.write('metaData.json')
zipf.write('recording.tmcpr')
os.remove('metaData.json')
os.remove('recording.tmcpr')
print('Finished!')
connection.close()
return should_restart
config, email, password = utils.load_config()
debug = config['debug_mode']
address = (config['ip'], int(config['port']))
while True:
try:
if not run(config, email, password, debug, address):
break
else:
print('Reconnecting...')
except Exception as e:
if debug:
print('Connection lost: ' + traceback.format_exc())
else:
print('Connection lost: ' + str(e))
if not config['auto_relog']:
break
def test():
config = load_config(FLAGS.config_file)
logger = get_logger(FLAGS.log_file)
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
files = os.listdir(FLAGS.test_b_filepath)
temp_dir = FLAGS.ner_result + '/submit_' + datetime.datetime.now(
).strftime('%Y%m%d_%H%M%S')
print(temp_dir)
os.makedirs(temp_dir)
for f1 in files:
f_name = f1.split(".")[0]
with codecs.open(os.path.join(FLAGS.test_b_filepath, f1), "r",
"utf-8") as f2:
s = f2.read()
line = []
sent = ''
leng = 0 # 当前处理句子的位置长度
for i in range(len(s)):
if s[i] != '。':
sent += s[i]
else:
sent += s[i]
line.append(sent)
sent = ''
f3 = codecs.open(os.path.join(temp_dir, f_name + ".ann"), "w",
"utf-8")
print(f3.name)
i = 0
for info in line:
result = model.evaluate_line(
sess, input_from_line(info, char_to_id), id_to_tag)
tag = result['entities']
# print(tag[0])
for char in tag:
sent = "T" + str(i + 1) + "\t" + char['type'] + " "
if char['word'].find("\n") == 0 or char['word'].find(
" ") == 0:
char_start = char['word'][0]
start = char['start'] + 1 + leng
word_new = char['word'].replace(char_start, "")
if char['word'].endswith(
"\n") or char['word'].endswith(" "):
char_end = char['word'][-1]
end = char['end'] - 1 + leng
sent = sent + str(start) + " " + str(
end) + "\t" + word_new.replace(
char_end, "")
elif 0 < char['word'].find("\n") < len(
char['word']):
j = char['word'].find("\n")
sent = sent + str(start) + " " + str(
char['start'] + leng + j) + ";" + str(
char['start'] + leng + j +
1) + " " + str(
char['end'] +
leng) + "\t" + word_new.replace(
"\n", " ")
else:
sent = sent + str(start) + " " + str(
char['end'] + leng) + "\t" + word_new
else:
start = char['start'] + leng
if char['word'].endswith(
"\n") or char['word'].endswith(" "):
char_end = char['word'][-1]
end = char['end'] - 1 + leng
sent = sent + str(start) + " " + str(
end) + "\t" + char['word'].replace(
char_end, "")
elif 0 < char['word'].find("\n") < len(
char['word']):
j = char['word'].find("\n")
sent = sent + str(start) + " " + str(
char['start'] + leng + j) + ";" + str(
char['start'] + leng + j +
1) + " " + str(char['end'] +
leng) + "\t" + char[
'word'].replace(
"\n", " ")
else:
sent = sent + str(start) + " " + str(
char['end'] + leng) + "\t" + char['word']
f3.write(sent + '\n')
i += 1
leng += len(info)
f3.close()
def get_corpus(corpus_config_path):
return KGCVAECorpus(config=utils.load_config(corpus_config_path))
def test_load_config(self):
expected_sp3url = "https://cats.oxfordfun.com"
result = utils.load_config("config.json")
self.assertEqual(expected_sp3url, result['sp3_url'])
import mesh_utils
import utils
parser = argparse.ArgumentParser(
description='Create point clouds from ShapeNet objs.')
parser.add_argument('config', type=str, help='Path to config file.')
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='store_true',
help='Be verbose.')
parser.set_defaults(verbose=False)
args = parser.parse_args()
# Read config.
cfg = utils.load_config(args.config)
shapenet_dir = cfg['data']['shapenet_dir']
out_dir = cfg['data']['out_dir']
meshes = cfg['data']['meshes']
num_points = cfg['data']['num_points']
bound = cfg['data']['bound']
num_train_transforms = cfg['data']['num_train_transforms']
num_test_transforms = cfg['data']['num_test_transforms']
rotation_max = cfg['data']['rotation_max']
translation_max = cfg['data']['translation_max']
# Setup out.
if not os.path.exists(out_dir):
os.mkdir(out_dir)
def single_agent():
config = load_config()
# num_agents = config['num_agents']
torch.set_num_threads(1)
env = GymEnv(config=config)
env.reset()
net = ActorCritic(True, config)
net.ActorNetwork.init_params()
net.CriticNetwork.init_params()
bwe = config['sending_rate'][config['default_bwe']]
i = 1
s_batch = []
r_batch = []
a_batch = []
# experience RTC if not forced to stop
ax = []
ay = []
plt.ion()
while True:
# todo: Agent interact with gym
state, reward, done, _ = env.step(bwe)
r_batch.append(reward)
action = net.predict(state)
bwe = config['sending_rate'][action]
a_batch.append(action)
s_batch.append(state)
# todo: need to be fixed
if done:
action = config['default_bwe']
bwe = config['sending_rate'][action]
# update network
net.getNetworkGradient(s_batch, a_batch, r_batch, done)
net.updateNetwork()
print('Network update.')
i += 1
ax.append(i)
# ay.append(entropy)
ay.append(reward)
plt.clf()
plt.plot(ax, ay)
plt.pause(0.1)
# s_batch.append(np.zeros(config['state_dim'], config['state_length']))
# a_batch.append(action)
env.reset()
print('Environment has been reset.')
print('Epoch {}, Reward: {}'.format(i - 1, reward))
if i % 100 == 0:
# print('Current BWE: ' + str(bwe))
torch.save(net.ActorNetwork.state_dict(),
config['model_dir'] + '/actor1_{}.pt'.format(str(i)))
torch.save(net.CriticNetwork.state_dict(),
config['model_dir'] + '/critic13m_{}.pt'.format(str(i)))
print('Model Restored.')
def evitram():
# Restore pretrained model
restorestr = pxfinestr.split('.meta')[0]
# Save model str
evitramstr = evitramfinestr.split('.meta')[0]
# Load pretrained evidence representations for all sources
K = []
for e in sys.argv[3:]:
cp2 = utils.load_config(e)
K.append(cp2.get('Experiment', 'PX_Z_TRAIN'))
sect = 'Experiment'
ev_paths = [cp.get(sect, i) for i in cp.options(sect) if 'evidence' in i]
if cp.get('Experiment', 'PREFIX') == 'MNIST':
evitram_dict = ConvAE.build_EviTRAM(cp, SEED)
else:
# Layerwise autoencoder number
ae_ids = [str(i) for i in xrange(cp.getint('Experiment', 'AENUM'))]
evitram_dict = SAE.build_EviTRAM(cp, ae_ids, SEED)
# Get variables to restore from pretrained model P(x) Encoder
var_list = tf.trainable_variables()
for ev_path_id, ev_path in enumerate(ev_paths):
if cp.get('Experiment', 'PREFIX') == 'MNIST':
# Prepare "restore" variable list
for v in var_list:
if v.name == 'Pre_Q' + str(ev_path_id) + '/kernel:0':
var_list.remove(v)
for v in var_list:
if v.name == 'Pre_Q' + str(ev_path_id) + '/bias:0':
var_list.remove(v)
else:
# Prepare "restore" variable list
for v in var_list:
if v.name == 'Pre_Q' + str(ev_path_id) + '/kernel:0':
var_list.remove(v)
for v in var_list:
if v.name == 'Pre_Q' + str(ev_path_id) + '/bias:0':
var_list.remove(v)
for v in var_list:
if v.name == 'Pre_Comp_Q' + str(ev_path_id) + '/kernel:0':
var_list.remove(v)
for v in var_list:
if v.name == 'Pre_Comp_Q' + str(ev_path_id) + '/bias:0':
var_list.remove(v)
##########################################################
# Tensorboard (comment / uncomment)
##########################################################
from datetime import datetime
now = datetime.utcnow().strftime("%m-%d_%H-%M:%S")
root_logdir = cp.get('Experiment', 'ModelOutputPath')
logdir = "{}/{}{}-{}/".format(
root_logdir,
cp.get('Experiment', 'PREFIX') + '_' +
cp.get('Experiment', 'Enumber') + '_cond', sys.argv[2], now)
tf.summary.scalar(name='cond loss', tensor=evitram_dict['evitram_loss'])
tf.summary.scalar(name='recon loss', tensor=evitram_dict['px_mse'])
summary = tf.summary.merge_all()
file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())
##########################################################
# Initialize & restore P(x) AE weights
init = tf.global_variables_initializer()
saver = tf.train.Saver(var_list)
saverCOND = tf.train.Saver()
# Task outcomes
EV = [np.load(i) for i in K]
# Start Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Init values
init.run()
# Restore finetuned model
saver.restore(sess, restorestr)
train_dict = {
'cp': cp,
'sess': sess,
'data': XX_full,
'sumr': summary,
'savestr': evitramstr,
'saver': saverCOND,
'fw': file_writer,
'EV': EV,
'ev_paths': ev_paths
}
if cp.get('Experiment', 'PREFIX') == 'MNIST':
ConvAE.evitram_train(train_dict, evitram_dict)
else:
SAE.evitram_train(train_dict, evitram_dict)
# Get batch size for batch output save
batch_size = train_dict['cp'].getint('Hyperparameters', 'BatchSize')
if cp.get('Experiment', 'PREFIX') == 'MNIST':
# Save hidden/output layer results for pipeline training
px_Z_latent = utils.run_OOM(sess,
evitram_dict['conv_in'],
XX_full,
evitram_dict['conv_z'],
batch_size=batch_size)
else:
px_Z_latent = utils.run_OOM(sess,
evitram_dict['sda_in'],
XX_full,
evitram_dict['sda_hidden'],
batch_size=batch_size)
# utils.save_OOM(sess, pae_dict['conv_in'], XX_full,
# pae_dict['conv_out'],
# path=cp.get('Experiment', 'PX_XREC_TRAIN'),
# batch_size=batch_size)
# Print clustering ACC
utils.log_accuracy(cp, YY_full, px_Z_latent, 'COND - ACC FULL', SEED)
# Print clustering NMI
utils.log_NMI(cp, YY_full, px_Z_latent, 'COND - NMI FULL', SEED)
sess.close()
'Trajectory type to use when rendering the headings. (Default: gt). If not gt, the trajectory file is taken as <args.out_dir>/<data_name>_<use_trajectory_type>.npy with files generated in ronin_lstm_tcn.py or ronin_resnet.py'
)
'''
Extra arguments
Set True: use_scheduler, quite (no output on stdout)
force_lr (force lr when a model is loaded from continue_from),
heading_norm (normalize heading),
separate_loss (report loss separately for logging)
float: dropout, max_ori_error (err. threshold for priority grv in degrees)
max_velocity_norm (filter outliers in training)
weights (array of float values)
'''
args, unknown_args = parser.parse_known_args()
np.set_printoptions(formatter={'all': lambda x: '{:.6f}'.format(x)})
args, kwargs = load_config(default_config_file, args, unknown_args)
if args.mode == "train" and kwargs.get('weights') and type(
kwargs.get('weights')) != list:
kwargs['weights'] = [
float(i) for i in kwargs.get('weights').split(',')
]
print(args, kwargs)
if args.mode == 'train':
train(args, **kwargs)
elif args.mode == 'test':
if not args.model_path:
raise ValueError("Model path required")
args.batch_size = 1
test(args, **kwargs)
def train():
# load data sets
datasets = load_sentences(FLAGS.train_file, FLAGS.lower)
random.shuffle(datasets)
train_sentences = datasets[:14000]
test_sentences = datasets[14000:]
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
char_to_id, _ = elmo_char_mapping(FLAGS.elmo_vocab)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i sentences in train / dev." %
(len(train_data), len(test_data)))
elmo_batcher = get_batcher()
train_manager = BatchManager(train_data, FLAGS.batch_size, elmo_batcher)
test_manager = BatchManager(test_data, FLAGS.batch_size, elmo_batcher)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto(allow_soft_placement=True)
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
elmo_model = load_elmo()
model = create_model(sess, Model, FLAGS.ckpt_path, elmo_model, config,
logger)
logger.info("start training")
loss = []
for i in range(FLAGS.max_epoch):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info(
"iteration:{} step:{}/{}, NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch, steps_per_epoch,
np.mean(loss)))
loss = []
best = evaluate(sess, model, "test", test_manager, id_to_tag,
logger)
# evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
self.detail_len = 0
self.piece1_len = 0
self.step = 0
self.recv_message()
class appServer(TCPServer):
def __init__(self, mysql_config):
TCPServer.__init__(self, )
self.__linkdb_logic = linkdbLogic.linkdbLogic(mysql_config)
def handle_stream(self, stream, address):
self.__linkdb_logic.mysql_ping()
Connection(stream, address, self.__linkdb_logic)
if __name__ == '__main__':
print "%s link_db_updater Server start ......" % (time.asctime(), )
logger = logging.getLogger()
hdlr = logging.FileHandler('./log/link_db_update.log')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
# load the config
my_config = utils.load_config(sys.argv[1])
server = appServer(my_config['mysql_config'])
server.listen(my_config['server_config']['port'])
IOLoop.instance().start()
from confluent_kafka import Producer
import datetime
import os
import pathlib
import requests
import subprocess
import time
from alert_broker_ztf import watchdog
from utils import init_db_sync, load_config, log, Mongo
""" load config and secrets """
config = load_config(config_file="config.yaml")["kowalski"]
class Program:
def __init__(self, group_name="FRITZ_TEST", group_nickname="Fritz"):
self.access_token = config["skyportal"]["token"]
self.base_url = (
f"{config['skyportal']['protocol']}://"
f"{config['skyportal']['host']}:{config['skyportal']['port']}"
)
self.headers = {"Authorization": f"token {self.access_token}"}
self.group_name = group_name
self.group_nickname = group_nickname
self.group_id, self.filter_id = self.create()
def get_groups(self):
resp = requests.get(
import numpy as np
from gensim.models import Doc2Vec
from utils import load_config, load_lab_seq_sp, embedding_to_file, labels_to_file
config = load_config("config.json")
dim = config["embedding_dimensions"]
model_sym_path = config["model_sym_path"]
model_SP_path = config["model_sp_path"]
training_data_file = config["training_data"]
embeddings_data_path = config["trained_embeddings_path"]
labels_data_path = config["labels"]
model_sym = Doc2Vec.load(model_sym_path)
model_SP = Doc2Vec.load(model_SP_path)
input_trajectories, input_sp, labels = load_lab_seq_sp(training_data_file)
labels_dict = dict.fromkeys(labels)
num_trajs = len(list(labels_dict))
print("Found {} unique user trajectories".format(num_trajs))
sum_vector = np.zeros(dim, dtype=np.float64)
index = 0
export_labels = []
total_labels = len(labels)
for label in labels:
if index % 500 == 0:
print("Evaluating traj {}/{} of user {}".format(index, total_labels, label))
# -*- coding:utf-8 -*-
import tensorflow as tf
import os
from collections import OrderedDict
from init_data import DataInitializer
from dataset import DataManager
from utils import create_model, init_logger, add_log, save_model, clean_model, load_config, ensure_dir
from model import Model
from engine import Engine
config = load_config('data/origin_data/config.json')
flags = tf.app.flags
flags.DEFINE_boolean('clean', True, 'Clean train folder')
flags.DEFINE_boolean('is_train', True, 'Whether train the model or evaluate')
flags.DEFINE_boolean('preprocess', False, 'Preprocess the data')
flags.DEFINE_integer('embedding_dim', config['embedding_dim'],
'Embedding size of words')
flags.DEFINE_integer('vocabulary_size', config['vocabulary_size'],
'Vocabulary size of corpus')
flags.DEFINE_float('dropout_keep', 0.5, 'Dropout rate')
flags.DEFINE_integer('batch_size', 160, 'Batch size')
flags.DEFINE_float('lr', 0.5, 'Learning rate')
flags.DEFINE_string('optimizer', 'sgd', 'Optimizer for training')
flags.DEFINE_integer('num_epochs', 30, 'Maximum training epochs')
flags.DEFINE_integer('steps_check', 5, 'Steps per checkpoint')
flags.DEFINE_integer('save_epoch', 2, 'Save model after n epochs')
flags.DEFINE_string(
'epoch_range',
('vaccine_year', './domain_weights/vaccine_year',
'./split_data_idx/vaccine_year/'),
('economy_year', './domain_weights/economy_year',
'./split_data_idx/economy_year/'),
('yelp_rest_year', './domain_weights/yelp_rest_year',
'./split_data_idx/yelp_rest_year/'),
('yelp_hotel_year', './domain_weights/yelp_hotel_year',
'./split_data_idx/yelp_hotel_year/'),
('amazon_year', './domain_weights/amazon_year',
'./split_data_idx/amazon_year/'),
('dianping_year', './domain_weights/dianping_year',
'./split_data_idx/dianping_year/'),
]
# load the configurations
config = utils.load_config('./config.ini')
"""
Loop through each dataset
"""
for datap in data_list:
print('Working on: -----------------' + datap[0])
valid_result = open('./results/hawkes/results_rmsprop.txt', 'a')
best_valid_f1 = 0
best_model = None
wt_iter = utils.load_weights(datap[1])
# number of predicted labels
if 'amazon' in datap[0] or 'yelp' in datap[0] or 'dianping' in datap[0]:
config['rnn']['pred_num'] = 3
else:
def main(conf: str, seed: int, gpu_index: int, data_path: str):
global DEVICE
conf = load_config(conf)
if seed is not None:
conf.seed = seed
if gpu_index is not None and DEVICE == torch.device('cuda'):
DEVICE = torch.device(f'cuda:{gpu_index}')
if data_path is not None:
conf['dataset']['params']['data_path'] = data_path
logger.info(DEVICE)
logger.info(conf)
set_seed(conf.seed)
from models import sub_task
tau4vec = set_task(conf.sub_task_params, 'tau4vec', sub_task)
logger.info('set_task: tau4vec')
set_seed(conf.seed)
higgsId = set_task(conf.sub_task_params, 'higgsId', sub_task)
logger.info('set_task: higgsId')
from models import MyDataset
from models import MyMetrics
set_seed(conf.seed)
dataset = set_module([MyDataset], conf, 'dataset')
set_seed(conf.seed)
dataloader = DataLoader(dataset, batch_size=100, shuffle=True)
logger.info('set dataloader')
# #########################################################################
# pre-train ###############################################################
# #########################################################################
logger.info('----- pretrain[0] start -----')
pretrain_conf = conf.sub_task_params.tau4vec.pretrain
for i, sub_model in enumerate(tau4vec):
logger.info(f'pretrain: [0][{i}]')
set_seed(conf.seed)
optimizer = set_module([optim],
pretrain_conf,
'optimizer',
params=sub_model.parameters())
loss_func = set_module([nn, MyLoss], pretrain_conf, 'loss_func')
metrics = set_module([MyMetrics], pretrain_conf, 'metrics')
activation = set_module([nn], pretrain_conf, 'activation')
input_key = pretrain_conf.data.input_key
target_key = pretrain_conf.data.target_key
patience = pretrain_conf.patience
tau4vec[i] = sub_task.pre_train(epochs=pretrain_conf.epochs,
model=sub_model,
dataloader=dataloader,
optimizer=optimizer,
loss_func=loss_func,
input_key=input_key,
target_key=target_key,
device=DEVICE,
patience=patience,
metrics=metrics,
activation=activation)
logger.info('----- pretrain[0] end -----')
logger.info('----- pretrain[1] start -----')
pretrain_conf = conf.sub_task_params.higgsId.pretrain
for i, sub_model in enumerate(higgsId):
logger.info(f'pretrain: [1][{i}]')
set_seed(conf.seed)
optimizer = set_module([optim],
pretrain_conf,
'optimizer',
params=sub_model.parameters())
loss_func = set_module([nn], pretrain_conf, 'loss_func')
metrics = set_module([MyMetrics], pretrain_conf, 'metrics')
activation = set_module([nn], pretrain_conf, 'activation')
input_key = pretrain_conf.data.input_key
target_key = pretrain_conf.data.target_key
patience = pretrain_conf.patience
higgsId[i] = sub_task.pre_train(epochs=pretrain_conf.epochs,
model=sub_model,
dataloader=dataloader,
optimizer=optimizer,
loss_func=loss_func,
input_key=input_key,
target_key=target_key,
device=DEVICE,
patience=patience,
metrics=metrics,
activation=activation)
logger.info('----- pretrain[1] end -----')
# #########################################################################
# #########################################################################
logger.info('copy the pretrain models')
pre_trained_tau4vec = set_task(conf.sub_task_params, 'tau4vec', sub_task)
pre_trained_higgsId = set_task(conf.sub_task_params, 'higgsId', sub_task)
pre_trained_model = [pre_trained_tau4vec, pre_trained_higgsId]
task = [tau4vec, higgsId]
for num_task, sub in enumerate(task):
for num_model in range(len(sub)):
pre_trained_model[num_task][num_model].load_state_dict(
deepcopy(task[num_task][num_model].state_dict()))
# #########################################################################
# #########################################################################
logger.info('----- SPOS-NAS start -----')
sposnas_conf = conf.SPOS_NAS
def make_output_dict():
return {
'X': [],
'AUC': {
f'{f}_{s}': []
for f, s in product(FIRST_MODEL_NAME, SECOND_MODEL_NAME)
},
'LOSS_1ST': {
f'{f}_{s}': []
for f, s in product(FIRST_MODEL_NAME, SECOND_MODEL_NAME)
},
'LOSS_2ND': {
f'{f}_{s}': []
for f, s in product(FIRST_MODEL_NAME, SECOND_MODEL_NAME)
},
'RATIO': {
f'{f}_{s}': []
for f, s in product(FIRST_MODEL_NAME, SECOND_MODEL_NAME)
},
'ONLY_PT_RATIO': {
f'{f}_{s}': []
for f, s in product(FIRST_MODEL_NAME, SECOND_MODEL_NAME)
},
}
# evaluate only pre-train model
loss_func = [
set_module([nn, MyLoss], sposnas_conf, 'loss_first'),
set_module([nn, MyLoss], sposnas_conf, 'loss_second')
]
loss_weight = [0.5, 0.5]
metrics = get_module([MyMetrics], 'Calc_Auc')()
from models.SPOS_NAS import SPOS
model = SPOS(task=task, loss_func=loss_func, loss_weight=loss_weight)
model.to(DEVICE)
logger.info('evaluate only pre-train model')
dummy = make_output_dict()
for now_choice in product(range(3), range(3)):
pre_train_result = evaluate(model, conf, dataloader, metrics, dummy,
now_choice)
output_dict = make_output_dict()
X_list = [0.0, 0.1, 0.5]
for X in (np.array(X_list)).round(10):
output_dict['X'].append(X)
logger.info(f'loss_ratio: {X:.6f} (loss_1*X + loss_2*(1-X)) start')
set_seed(conf.seed)
def initialize_pretrain_weight():
logger.info('load pretrain models...')
for num_task, sub in enumerate(task):
for num_model in range(len(sub)):
task[num_task][num_model].load_state_dict(
deepcopy(pre_trained_model[num_task]
[num_model].state_dict()))
logger.info('load pretrain models done')
logger.info('set model parameters...')
loss_func = [
set_module([nn, MyLoss], sposnas_conf, 'loss_first'),
set_module([nn, MyLoss], sposnas_conf, 'loss_second')
]
loss_weight = [X, 1. - X]
metrics = get_module([MyMetrics], 'Calc_Auc')()
for now_choice in product(range(3), range(3)):
initialize_pretrain_weight()
model = SPOS(task=task,
loss_func=loss_func,
loss_weight=loss_weight)
model.to(DEVICE)
optimizer = set_module([optim],
sposnas_conf,
'optimizer',
params=model.parameters())
scheduler = set_module([optim.lr_scheduler],
sposnas_conf,
'scheduler',
optimizer=optimizer)
logger.info('set model parameters done')
logger.info('fit model...')
model.fit(epochs=sposnas_conf.epochs,
dataloader=dataloader,
device=DEVICE,
optimizer=optimizer,
scheduler=scheduler,
patience=sposnas_conf.patience,
choice=now_choice)
logger.info('fit model done')
logger.info('eval model...')
output_dict = evaluate(model, conf, dataloader, metrics,
output_dict, now_choice)
logger.info('eval model done')
logger.info(f'seed: {conf.seed}/ pretrain result: {pre_train_result}')
logger.info(f'seed: {conf.seed}/ final result: {output_dict}')
logger.info('all train and eval step are done')
logger.info('plot results...')
logger.info('plot auc...')
import matplotlib.pyplot as plt
plt.style.use('seaborn-darkgrid')
import pandas as pd
df = pd.DataFrame(output_dict['AUC'], index=output_dict['X'])
df = df.rename(
columns={
f'{f}_{s}': f'{f}:{s}'
for f, s in product(FIRST_MODEL_NAME, SECOND_MODEL_NAME)
})
df.plot()
plt.xlabel('X')
plt.ylabel('AUC')
plt.savefig(f'grid_auc_{conf.seed}.png')
plt.close()
logger.info('plot loss_2ND...')
import matplotlib.pyplot as plt
plt.style.use('seaborn-darkgrid')
df = pd.DataFrame(output_dict['LOSS_2ND'], index=output_dict['X'])
df = df.rename(
columns={
f'{f}_{s}': f'{f}:{s}'
for f, s in product(FIRST_MODEL_NAME, SECOND_MODEL_NAME)
})
df.plot()
plt.xlabel('X')
plt.ylabel('LOSS_2ND')
plt.savefig(f'grid_loss_2nd_{conf.seed}.png')
plt.close()
logger.info('plot loss_1ST...')
import matplotlib.pyplot as plt
plt.style.use('seaborn-darkgrid')
df = pd.DataFrame(output_dict['LOSS_1ST'], index=output_dict['X'])
df = df.rename(
columns={
f'{f}_{s}': f'{f}:{s}'
for f, s in product(FIRST_MODEL_NAME, SECOND_MODEL_NAME)
})
df.plot()
plt.xlabel('X')
plt.ylabel('LOSS_1ST')
plt.savefig(f'grid_loss_1st_{conf.seed}.png')
plt.close()
logger.info('plot ratios...')
import matplotlib.pyplot as plt
plt.style.use('seaborn-darkgrid')
df = pd.DataFrame(output_dict['ONLY_PT_RATIO'], index=output_dict['X'])
df = df.rename(
columns={
f'{f}_{s}': f'{f}:{s}'
for f, s in product(FIRST_MODEL_NAME, SECOND_MODEL_NAME)
})
df.plot()
plt.ylabel('ratio')
plt.savefig(f'grid_only_pt_ratio_{conf.seed}.png')
plt.close()
import matplotlib.pyplot as plt
plt.style.use('seaborn-darkgrid')
df = pd.DataFrame(output_dict['RATIO'], index=output_dict['X'])
df = df.rename(
columns={
f'{f}_{s}': f'{f}:{s}'
for f, s in product(FIRST_MODEL_NAME, SECOND_MODEL_NAME)
})
df.plot()
plt.ylabel('ratio')
plt.savefig(f'grid_ratio_{conf.seed}.png')
plt.close()
logger.info('plot results done')
def realtime_price(ticker, fx=None, source=None, parsed=True):
'''
Gets realtime price from first provider available and returns
result = {
'symbol': ,
'name': ,
'price': ,
'fx': ,
'time': ,
'timezone':
'source':
}
'''
if fx is None:
config = load_config()
fx = config['PORTFOLIO']['base_fx']
if fx == 'USD':
fxrate = 1
else:
from pricing_engine.alphavantage import realtime as aa_realtime
fxrate = aa_realtime(fx)
fxrate = parseNumber(fxrate['price'])
ticker = ticker.replace(' ', '')
if source and type(source) != list:
raise TypeError("source has to be a list of strings - can be one string inside a list")
try:
source_list = mapping[ticker]
except KeyError:
source_list = [
'cryptocompare',
'alphavantage_currency',
'alphavantage_global',
'twelvedata',
'fmp'
]
from pricing_engine.alphavantage import realtime as aa_realtime
from pricing_engine.cryptocompare import realtime as cc_realtime
from pricing_engine.fmp import realtime as fmp_realtime
from pricing_engine.twelvedata import realtime as td_realtime
results = None
# Gets from each source
for src in source_list:
if src == 'alphavantage_currency':
results = aa_realtime(ticker, 'USD', 'CURRENCY_EXCHANGE_RATE', parsed=parsed)
if src == 'alphavantage_global':
results = aa_realtime(ticker, 'USD', 'GLOBAL_QUOTE', parsed=parsed)
if src == 'cryptocompare':
results = cc_realtime(ticker, 'USD', parsed=parsed)
if src == 'fmp':
results = fmp_realtime(ticker, parsed=parsed)
if src == 'twelvedata':
results = td_realtime(ticker, parsed=parsed)
# Check if data is valid
if results is not None:
if parsed and 'price' in results:
if results['price'] is not None:
if isinstance(results['time'], str):
results['time'] = parser.parse(results['time'])
results['price'] = parseNumber(results['price'])
results['price'] = (
results['price'] / fxrate)
return (results)
return (results)
def train():
# load data sets
# train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
# dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
all_train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
train_sentences, dev_sentences = split_train_dev(all_train_sentences)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# update_tag_scheme(dev_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
# dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars_train = char_mapping(all_train_sentences,
FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(all_train_sentences,
FLAGS.lower)
# _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(all_train_sentences)
# _t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# nlp = StanfordCoreNLP(r'E:\DC\dataset\泰一指尚评测数据\stanford-corenlp-full-2017-06-09')
#l_sorted_lexcion = load_lexcion(FLAGS.lexcion_file, nlp)
l_sorted_lexcion = []
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
l_sorted_lexcion, FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
l_sorted_lexcion, FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
l_sorted_lexcion, FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
max_len = max(
[len(sentence[0]) for sentence in train_data + test_data + dev_data])
train_manager = BatchManager(train_data, FLAGS.batch_size, max_len)
dev_manager = BatchManager(dev_data, 800, max_len)
test_manager = BatchManager(test_data, 800, max_len)
# random.shuffle(train_data)
# pad_test_data = pad_data(test_data)
# pad_dev_data = pad_data(dev_data)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id, max_len)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(FLAGS.max_epoch):
random.shuffle(train_data)
pad_train_data = pad_data(train_data, max_len)
strings, chars, lexcion_teatures, pos_ids, dep_ids, head_ids, targets = pad_train_data
for j in range(0, len(strings), FLAGS.batch_size):
batch = [
strings[j:j + FLAGS.batch_size],
chars[j:j + FLAGS.batch_size],
lexcion_teatures[j:j + FLAGS.batch_size],
pos_ids[j:j + FLAGS.batch_size],
dep_ids[j:j + FLAGS.batch_size],
head_ids[j:j + FLAGS.batch_size],
targets[j:j + FLAGS.batch_size]
]
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"AS loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger, i)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def run_to_task(task_to):
import general_utils
from general_utils import RuntimeDeterminedEnviromentVars
import models.architectures as architectures
from data.load_ops import resize_rescale_image
import utils
from data.task_data_loading import load_and_specify_preprocessors_for_representation_extraction
import lib.data.load_ops as load_ops
from importlib import reload
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
# for arch in ['regular', 'shallow', 'dilated_shallow', 'dilated_regular']:
arch = args.arch
data_amount = args.data
if not args.no_regenerate_data:
#if False:
all_outputs = {}
pickle_dir = 'viz_{task_to}_transfer_{hs}_{arch}.pkl'.format(
arch=arch, hs=args.hs, task_to=task_to)
subprocess.call(
"aws s3 cp s3://task-preprocessing-512-oregon/visualizations/transitive_viz/viz_{}.pkl {}"
.format(task_to, pickle_dir),
shell=True)
import os
if os.path.isfile(pickle_dir):
with open(pickle_dir, 'rb') as fp:
all_outputs = pickle.load(fp)
global list_of_src_tasks
for i, task_from in enumerate(list_of_src_tasks):
task_key = task_from.split('/')[-1]
if task_key in all_outputs:
print("{} already exists....\n\n\n".format(task_key))
continue
print("Doing from {task_from} to {task_to}".format(
task_from=task_key, task_to=task_to))
general_utils = importlib.reload(general_utils)
tf.reset_default_graph()
training_runners = {
'sess': tf.InteractiveSession(),
'coord': tf.train.Coordinator()
}
CONFIG_DIR = task_from
print(CONFIG_DIR)
############## Load Configs ##############
cfg = utils.load_config(CONFIG_DIR, nopause=True)
RuntimeDeterminedEnviromentVars.register_dict(cfg)
if args.second_order and not to_flip_dict[task]:
cfg['val_representations_file'] = cfg[
'val_representations_file'][::-1]
cfg['num_epochs'] = 1
cfg['randomize'] = False
root_dir = cfg['root_dir']
cfg['num_read_threads'] = 1
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train'
#'time'
))
# print(cfg['model_path'])
if cfg['model_path'] is None and task == 'random':
cfg['model_path'] = tf.train.latest_checkpoint(
os.path.join(cfg['log_root'], 'logs', 'slim-train',
'time'))
if cfg['model_path'] is None:
continue
############## Set Up Inputs ##############
# tf.logging.set_verbosity( tf.logging.INFO )
inputs = utils.setup_input_transfer(
cfg, is_training=ON_TEST_SET, use_filename_queue=False
) # is_training determines whether to use train/validaiton
RuntimeDeterminedEnviromentVars.load_dynamic_variables(inputs, cfg)
RuntimeDeterminedEnviromentVars.populate_registered_variables()
start_time = time.time()
# utils.print_start_info( cfg, inputs[ 'max_steps' ], is_training=False )
############## Set Up Model ##############
model = utils.setup_model(inputs, cfg, is_training=True)
m = model['model']
model['saver_op'].restore(training_runners['sess'],
cfg['model_path'])
############## Start dataloading workers ##############
data_prefetch_init_fn = utils.get_data_prefetch_threads_init_fn_transfer(
inputs, cfg, is_training=ON_TEST_SET, use_filename_queue=False)
prefetch_threads = threading.Thread(
target=data_prefetch_init_fn,
args=(training_runners['sess'], training_runners['coord']))
prefetch_threads.start()
############## Run First Batch ##############
(
input_batch,
representation_batch,
target_batch,
data_idx,
encoder_output,
predicted,
loss,
) = training_runners['sess'].run([
m.input_images, m.input_representations, m.decoder.targets,
model['data_idxs'], m.encoder_output, m.decoder.decoder_output,
m.total_loss
])
if task_to == 'segment2d' or task_to == 'segment25d':
from sklearn.decomposition import PCA
x = np.zeros((32, 256, 256, 3), dtype='float')
for i in range(predicted.shape[0]):
embedding_flattened = np.squeeze(predicted[i]).reshape(
(-1, 64))
pca = PCA(n_components=3)
pca.fit(embedding_flattened)
lower_dim = pca.transform(embedding_flattened).reshape(
(256, 256, -1))
lower_dim = (lower_dim - lower_dim.min()) / (
lower_dim.max() - lower_dim.min())
x[i] = lower_dim
predicted = x
if task_to == 'segmentsemantic_rb':
predicted = np.argmax(predicted, axis=-1)
############## Clean Up ##############
training_runners['coord'].request_stop()
training_runners['coord'].join()
# if os.path.isfile(pickle_dir):
# with open(pickle_dir, 'rb') as fp:
# all_outputs = pickle.load(fp)
############## Store to dict ##############
to_store = {
'data_idx': data_idx,
'output': predicted,
'loss': loss
}
all_outputs[task_key] = to_store
# os.system("sudo cp {d} /home/ubuntu/s3/model_log".format(d=pickle_dir))
############## Reset graph and paths ##############
tf.reset_default_graph()
training_runners['sess'].close()
#print(sys.modules.keys())
#del sys.modules[ 'config' ]
sys.path = remove_dups(sys.path)
print('Current Directory: ', os.getcwd())
pickle_dir = 'viz_{task_to}_transfer_{hs}_{arch}.pkl'.format(
arch=arch, hs=args.hs, task_to=task_to)
with open(pickle_dir, 'wb') as fp:
pickle.dump(all_outputs, fp)
subprocess.call(
"aws s3 cp {} s3://task-preprocessing-512-oregon/visualizations/transitive_viz/viz_{}.pkl"
.format(pickle_dir, task_to),
shell=True)
# Run jupyter nb
print('Running Jupyter Notebooks...')
#os.makedirs("/home/ubuntu/task-taxonomy-331b/notebooks/transfer_viz/transfer_{hs}_{arch}".format(hs=args.hs, arch=arch), exist_ok=True)
notebook_path = '/home/ubuntu/task-taxonomy-331b/notebooks/transfer_viz/Visual_{task_to}'.format(
task_to=task_to)
subprocess.call("jupyter nbconvert \
--execute {notebook_path}.ipynb \
--to html \
--ExecutePreprocessor.kernel_name=python3 \
--ExecutePreprocessor.timeout=1200 ".format(
notebook_path=notebook_path, arch=arch, hs=args.hs, task_to=task_to),
shell=True)
subprocess.call(
"aws s3 cp {}.html s3://task-preprocessing-512-oregon/visualizations/{}/"
.format(notebook_path, TRANSFER_TYPE),
shell=True)
