def Export(self, filename, color=True, page=0, dpi=100, antialias=True, quality=85, backcolor='#ffffff00', pdfdpi=150, svgtextastext=False): """Export plot to filename. color is True or False if color is requested in output file page is the pagenumber to export dpi is the number of dots per inch for bitmap output files antialias antialiases output if True quality is a quality parameter for jpeg output backcolor is the background color for bitmap files, which is a name or a #RRGGBBAA value (red, green, blue, alpha) pdfdpi is the dpi to use when exporting eps or pdf files svgtextastext: write text in SVG as text, rather than curves """ e = export.Export(self.document, filename, page, color=color, bitmapdpi=dpi, antialias=antialias, quality=quality, backcolor=backcolor, pdfdpi=pdfdpi, svgtextastext=svgtextastext) e.export()
def admin_export_state_handler(update: Update, context: CallbackContext) -> int: message = helpers.get_message(update) user = message.from_user logger.info(f'@{user.username} exported survey results') export_dir_path = export.Export().export_to_html( user.id, should_remove_original_files=False) export_dir_path = os.path.normpath(export_dir_path) archive_file_name = os.path.split(export_dir_path)[-1] zip_file = shutil.make_archive( archive_file_name, 'zip', root_dir=export_dir_path, base_dir='.', ) reply_keyboard = [['Вернуться в главное меню админки']] keyboard_markup = ReplyKeyboardMarkup(reply_keyboard, one_time_keyboard=True) with open(zip_file, 'rb') as f: message.reply_document(f, reply_markup=keyboard_markup, reply_to_message_id=message.message_id) os.remove(zip_file) return states.ADMIN_EXPORT_RESULT_STATE
def test_save_file(self, mocker): exportInstance = export.Export(self.prog) exportStub = stubExport() path = ('/home/cameron/test.csv', 'null') # mocker.spy(exportInstance, 'to_csv') mocker.spy(exportInstance, 'to_json') mocker.spy(exportInstance, 'to_xml') mocker.spy(exportInstance, 'to_csv') exportInstance.save_file(path, exportStub.data, 'CSV') exportInstance.save_file(path, exportStub.data, 'JSON') exportInstance.save_file(path, exportStub.data, 'XML') assert (exportInstance.to_csv.call_count == 1) assert (exportInstance.to_json.call_count == 1) assert (exportInstance.to_xml.call_count == 1)
import export import IPython lr = 0.03 gamma = 0.0 sample_density = 20 group_num_particles = sample_density**2 goal_pos = np.array([1.4, 0.4]) goal_range = np.array([0.0, 0.00]) batch_size = 1 actuation_strength = 4 config = 'B' exp = export.Export('walker_data') # Robot B num_groups = 7 group_offsets = [(0, 0), (0.5, 0), (0, 1), (1, 1), (2, 1), (2, 0), (2.5, 0)] group_sizes = [(0.5, 1), (0.5, 1), (1, 1), (1, 1), (1, 1), (0.5, 1), (0.5, 1)] actuations = [0, 1, 5, 6] fixed_groups = [] head = 3 gravity = (0, -2) num_particles = group_num_particles * num_groups def particle_mask(start, end): r = tf.range(0, num_particles)
def main(): parser = ArgumentParser(description='PGObserver Frontend') parser.add_argument( '-c', '--config', help= 'Path to yaml config file with datastore connect details. See pgobserver_frontend.example.yaml for a sample file. \ Certain values can be overridden by ENV vars PGOBS_HOST, PGOBS_DBNAME, PGOBS_USER, PGOBS_PASSWORD [, PGOBS_PORT]' ) parser.add_argument( '--s3-config-path', help= 'Path style S3 URL to a key that holds the config file. Or PGOBS_CONFIG_S3_BUCKET env. var', metavar='https://s3-region.amazonaws.com/x/y/file.yaml', default=os.getenv('PGOBS_CONFIG_S3_BUCKET')) parser.add_argument( '-p', '--port', help='Web server port. Overrides value from config file', type=int) args = parser.parse_args() settings = collections.defaultdict(dict) if args.s3_config_path: # S3 has precedence if specified import aws_s3_configreader settings = aws_s3_configreader.get_config_as_dict_from_s3_file( args.s3_config_path) elif args.config: args.config = os.path.expanduser(args.config) if not os.path.exists(args.config): print 'WARNING. Config file {} not found! exiting...'.format( args.config) return print "trying to read config file from {}".format(args.config) with open(args.config, 'rb') as fd: settings = yaml.load(fd) # Make env vars overwrite yaml file, to run via docker without changing config file settings['database']['host'] = (os.getenv('PGOBS_HOST') or settings['database'].get('host')) settings['database']['port'] = (os.getenv('PGOBS_PORT') or settings['database'].get('port') or 5432) settings['database']['name'] = (os.getenv('PGOBS_DATABASE') or settings['database'].get('name')) settings['database']['frontend_user'] = ( os.getenv('PGOBS_USER') or settings['database'].get('frontend_user')) settings['database']['password'] = ( os.getenv('PGOBS_PASSWORD') or settings['database'].get('frontend_password')) if not (settings['database'].get('host') and settings['database'].get('name') and settings['database'].get('frontend_user')): print 'Mandatory datastore connect details missing!' print 'Check --config input or environment variables: PGOBS_HOST, PGOBS_DATABASE, PGOBS_USER, PGOBS_PASSWORD [, PGOBS_PORT]' print '' parser.print_help() return conn_string = ' '.join(( 'dbname=' + settings['database']['name'], 'host=' + settings['database']['host'], 'user='******'database']['frontend_user'], 'port=' + str(settings['database']['port']), )) print 'Setting connection string to ... ' + conn_string # finished print conn_string to the world, password can be added conn_string = conn_string + ' password='******'database'][ 'frontend_password'] datadb.setConnectionString(conn_string) current_dir = os.path.dirname(os.path.abspath(__file__)) conf = { 'global': { 'server.socket_host': '0.0.0.0', 'server.socket_port': args.port or settings.get('frontend', {}).get('port') or 8080 }, '/': { 'tools.staticdir.root': current_dir, 'request.dispatch': HostIdAndShortnameDispatcher() }, '/healthcheck': { 'tools.sessions.on': False }, '/static': { 'tools.staticdir.dir': 'static', 'tools.staticdir.on': True, 'tools.sessions.on': False }, '/manifest.info': { 'tools.staticfile.on': True, 'tools.staticfile.filename': os.path.join(current_dir, '..', 'MANIFEST.MF'), 'tools.auth_basic.on': False, 'tools.sessions.on': False }, } tplE.setup( settings) # setup of global variables and host data for usage in views root = welcomefrontend.WelcomeFrontend() root.host = monitorfrontend.MonitorFrontend() root.report = report.Report() root.export = export.Export() root.perftables = performance.PerfTables() root.perfapi = performance.PerfApi() root.perfindexes = performance.PerfIndexes() root.perfschemas = performance.PerfUnusedSchemas() root.perflocks = performance.PerfLocksReport() root.perfstatstatements = performance.PerfStatStatementsReport() root.perfbloat = performance.PerfBloat() root.sprocs = sprocsfrontend.SprocFrontend() root.tables = tablesfrontend.TableFrontend() root.indexes = indexesfrontend.IndexesFrontend() root.hosts = hostsfrontend.HostsFrontend() root.api = api.Root( root ) # JSON api exposure, enabling integration with other monitoring tools root.healthcheck = Healthcheck() if settings.get('oauth', {}).get('enable_oauth', False): print 'switching on oauth ...' import oauth root.oauth = oauth.Oauth(settings['oauth']) cherrypy.config.update({ 'tools.oauthtool.on': True, 'tools.sessions.on': True, 'tools.sessions.timeout': settings['oauth'].get('session_timeout', 43200) }) cherrypy.quickstart(root, config=conf)
lr = 1 gamma = 0.0 sample_density = 30 group_num_particles = sample_density**3 goal_pos = np.array([1.4, 0.4, 0.5]) goal_range = np.array([0.0, 0.0, 0.0]) batch_size = 1 actuation_strength = 2 config = 'C' exp = export.Export('walker3d') # Robot B if config == 'B': num_groups = 7 group_offsets = [(0, 0, 0), (0.5, 0, 0), (0, 1, 0), (1, 1, 0), (2, 1, 0), (2, 0, 0), (2.5, 0, 0)] group_sizes = [(0.5, 1, 1), (0.5, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (0.5, 1, 1), (0.5, 1, 1)] actuations = [0, 1, 5, 6] fixed_groups = [] head = 3 gravity = (0, -2, 0) #TODO: N-Ped #Robot C
import export import IPython lr = 0.03 gamma = 0.0 sample_density = 20 group_num_particles = sample_density**2 goal_pos = np.array([1.4, 0.4]) goal_range = np.array([0.0, 0.00]) batch_size = 1 actuation_strength = 3 config = 'B' exp = export.Export('walker_2d') # Robot B num_groups = 7 group_offsets = [(0, 0), (0.5, 0), (0, 1), (1, 1), (2, 1), (2, 0), (2.5, 0)] group_sizes = [(0.5, 1), (0.5, 1), (1, 1), (1, 1), (1, 1), (0.5, 1), (0.5, 1)] actuations = [0, 1, 5, 6] fixed_groups = [] head = 3 gravity = (0, 0) num_particles = group_num_particles * num_groups def particle_mask(start, end): r = tf.range(0, num_particles)
def save(self): export_result = export.Export(self, self.dirty, self.lastDirectory, None) export_result.to_json()
def export_to_word(self): export_result = export.Export(self, self.dirty, self.lastDirectory, self.final_building) export_result.to_word()
import IPython lr = 0.03 gamma = 0.0 sample_density = 40 group_num_particles = sample_density**2 goal_pos = np.array([0.5, 0.6]) goal_range = np.array([0.1, 0.1]) batch_size = 1 actuation_strength = 4 multi_target = True config = 'B' exp = export.Export('finger_data') # Robot B num_groups = 3 group_offsets = [(1, 0), (1.5, 0), (1, 2)] group_sizes = [(0.5, 2), (0.5, 2), (1, 1)] actuations = [0, 1] head = 2 gravity = (0, 0) num_particles = group_num_particles * num_groups def particle_mask(start, end): r = tf.range(0, num_particles) return tf.cast(tf.logical_and(start <= r, r < end), tf.float32)[None, :]
from vector_math import * import export import IPython lr = 0.03 gamma = 0.0 sample_density = 20 group_num_particles = sample_density**2 goal_pos = np.array([1.4, 0.4]) goal_range = np.array([0.0, 0.00]) batch_size = 1 config = 'B' exp = export.Export('walker_video') # Robot B num_groups = 1 group_offsets = [(1, 1)] group_sizes = [(0.5, 1)] actuations = [0] fixed_groups = [] head = 0 gravity = (0, 0) num_particles = group_num_particles * num_groups def particle_mask(start, end): r = tf.range(0, num_particles)
evaluate = False lr = 10 gamma = 0.0 sample_density = 15 group_num_particles = sample_density**3 goal_pos = np.array([1.4, 0.4, 0.5]) goal_range = np.array([0.0, 0.0, 0.0]) batch_size = 1 actuation_strength = 4 config = 'C' exp = export.Export('crawler3d') # Robot B if config == 'B': num_groups = 7 group_offsets = [(0, 0, 0), (0.5, 0, 0), (0, 1, 0), (1, 1, 0), (2, 1, 0), (2, 0, 0), (2.5, 0, 0)] group_sizes = [(0.5, 1, 1), (0.5, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1), (0.5, 1, 1), (0.5, 1, 1)] actuations = [0, 1, 5, 6] fixed_groups = [] head = 3 gravity = (0, -2, 0) #TODO: N-Ped #Robot C
import tensorflow.contrib.layers as ly from vector_math import * import IPython import copy import pygmo as pg #import pygmo_plugins_nonfree as ppnf import export def flatten_vectors(vectors): return tf.concat([tf.squeeze(ly.flatten(vector)) for vector in vectors], 0) lr = 1.0 exp = export.Export('arm3d') goal_range = 0.0 batch_size = 1 actuation_strength = 8 use_pygmo = True num_steps = 600 # Finger num_links = 2 num_acts = int(num_steps // num_links // 10) #TBH this is just to keep the number of variables tame sample_density = int(10 // (np.sqrt(num_links))) group_num_particles = sample_density**3
import sys sys.path.append('..') import random import time from simulation import Simulation, get_bounding_box_bc from time_integration import UpdatedSimulationState import tensorflow as tf import numpy as np from IPython import embed import export exp = export.Export('rolling_acc') def main(sess): batch_size = 1 gravity = (0, -1) # gravity = (0, 0) N = 5 dR = 0.2 R = (N - 1) * dR dC = 1.6 num_particles = int(((N - 1) * dC + 1)**2) steps = 1000 dt = 5e-3 goal_range = 0.15 res = (45, 30) bc = get_bounding_box_bc(res)
def main(sess): batch_size = 1 gravity = (0, -1.) # gravity = (0, 0) N = 10 dR = 0.1 R = (N - 1) * dR dC = 1.6 num_particles = int(((N - 1) * dC + 1) ** 2) steps = 1000 dt = 1e-2 goal_range = 0.15 res = (90, 60) dx = 1. / res[0] max_speed = 0.3 exp = export.Export('rolling_acc_{}'.format(max_speed)) lef = 2.36 rig = 9. temp_f = lambda x: 2 * x - 1 if x > 0.5 else -2 * x + 1 temp_f_ = lambda x: 2 if x > 0.5 else -2 boundary = lambda x: ((temp_f(x / res[0]) - 0.5) * difficulty + 0.5) * res[1] boundary_ = lambda x: temp_f_(x / res[0]) / res[0] * res[1] * difficulty boundary = lambda x: (np.sin((rig - lef) * x / res[0] + lef) * difficulty + 1) * res[1] / 2 boundary_ = lambda x: (rig - lef) / 2 * difficulty * res[1] * np.cos((rig - lef) * x / res[0] + lef) / res[0] tf_boundary = lambda x: (tf.sin((rig - lef) * x + lef) * difficulty + 1) * res[1] / 2 tf_boundary_ = lambda x: (rig - lef) / 2 * difficulty * res[1] * tf.cos((rig - lef) * x + lef) / res[0] bc = get_new_bc(res, boundary = boundary, boundary_ = boundary_) lr = 1 goal = tf.placeholder(dtype=tf.float32, shape=[batch_size, 2], name='goal') def state_tensor(state): cm = state.center_of_mass() mv = tf.reduce_mean(state.velocity, axis = 2) time1 = tf.sin(tf.cast(state.step_count, dtype = tf.float32) / 100) time2 = tf.sin(tf.cast(state.step_count, dtype = tf.float32) / 31) time3 = tf.sin(tf.cast(state.step_count, dtype = tf.float32) / 57) time4 = tf.sin(tf.cast(state.step_count, dtype = tf.float32) / 7) lis = [mv, cm, time1, time2, time3, time4] tensor = tf.concat([tf.reshape(t, shape = [batch_size, -1]) for t in lis], axis = 1) return tensor def norm_tensor(state): return (state_tensor(state) - model.get_bn_mean()) ** 2 def F_controller(state): # F = state.position - state.center_of_mass()[:, :, None] # F = tf.stack([F[:, 1], -F[:, 0]], axis = 1) # F = tf.constant([1, 2 / res[0] * res[1]])[None, :, None] # accelerate cm = state.center_of_mass() k = tf_boundary_(cm[:, 0]) L = (k ** 2 + 1) ** 0.5 dx = tf.reciprocal(L) dy = k / L F = tf.stack([dx, dy], axis = 1) * max_speed # inputs inputs = state_tensor(state) # network direction = model(inputs) ''' direction = thb[state.step_count] ''' ''' # best solution vx = tf.reduce_mean(state.velocity, axis = 2)[:, 0] direction = tf.cast(vx > 0, dtype = tf.float32) * 2 - 1 ''' return (F * direction)[:, :, None] sim = Simulation( dt=dt, num_particles=num_particles, grid_res=res, bc=bc, gravity=gravity, m_p=0.5, V_p=0.5, E = 1, nu = 0.3, dx = dx, sess=sess, use_visualize = True, F_controller = F_controller, part_size = 10) position = np.zeros(shape=(batch_size, num_particles, 2)) # velocity_ph = tf.constant([0.2, 0.3]) velocity_ph = tf.constant([0, 0], dtype = tf.float32) velocity = velocity_ph[None, :, None] + tf.zeros( shape=[batch_size, 2, num_particles], dtype=tf.float32) for b in range(batch_size): dx, dy = 15.81, 7.2 cnt = 0 las = 0 for i in range(N): l = int((dC * i + 1) ** 2) l, las = l - las, l print(l) dth = 2 * np.pi / l dr = R / (N - 1) * i theta = np.pi * 2 * np.random.random() for j in range(l): theta += dth x, y = np.cos(theta) * dr, np.sin(theta) * dr position[b, cnt] = ((dx + x) / 45, (dy + y) / 45) cnt += 1 position = np.array(position).swapaxes(1, 2) initial_state = sim.get_initial_state( position=position, velocity=velocity) state_sum = tf.reduce_mean(sim.stepwise_sym(state_tensor), axis = 0) state_norm = tf.reduce_mean(sim.stepwise_sym(norm_tensor), axis = 0) final_position = sim.initial_state.center_of_mass() final_velocity = tf.reduce_mean(sim.initial_state.velocity, axis = 2) final_F = tf.reduce_mean(sim.updated_state.F, axis = 2) loss = tf.reduce_sum((final_position - goal) ** 2) sim.add_point_visualization(pos = final_position, color = (1, 0, 0), radius = 3) sim.add_point_visualization(pos = goal, color = (0, 1, 0), radius = 3) sim.add_vector_visualization(pos=final_position, vector=final_velocity, color=(0, 0, 1), scale=50) sim.add_vector_visualization(pos=final_position, vector=final_F, color=(1, 0, 1), scale=500) trainables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) sim.set_initial_state(initial_state = initial_state) sym = sim.gradients_sym(loss, variables = trainables) goal_input = np.array([[37.39 / 45, 25. / 45]], dtype=np.float32) grad_ph = [ tf.placeholder(shape = v.shape, dtype = tf.float32) for v in trainables ] velocity = [ tf.Variable(np.zeros(shape = v.shape, dtype = np.float32), trainable=False) for v in trainables ] gradient_descent = [ v.assign(v - lr * g) for v, g in zip(trainables, grad_ph) ] momentum_beta = 0.9 momentum = [ v.assign(v * momentum_beta + g * (1 - momentum_beta)) for v, g in zip(velocity, grad_ph) ] + [ v.assign(v - lr * g) for v, g in zip(trainables, velocity) ] sess.run(tf.global_variables_initializer()) flog = open('rolling_{}.log'.format(max_speed), 'w') for i in range(100000): t = time.time() print('Epoch {:5d}, learning rate {}'.format(i, lr)) tt = time.time() memo = sim.run( initial_state = initial_state, num_steps = steps, iteration_feed_dict = {goal: goal_input}, loss = loss, stepwise_loss = [state_sum, state_norm]) print('forward', time.time() - tt) if False:# i % 10 == 0: tt = time.time() sim.visualize(memo, show = False, interval = 1, export = exp) print('visualize', time.time() - tt) if memo.loss < 1e-3: break tt = time.time() grad = sim.eval_gradients(sym=sym, memo=memo) print('eval_gradients', time.time() - tt) tt = time.time() grad_feed_dict = {} tot = 0. for gp, g in zip(grad_ph, grad): grad_feed_dict[gp] = g * (np.random.random(g.shape) < 0.8).astype('float32') tot += (g ** 2).sum() # if i % 10 == 0: # grad_feed_dict[gp] += (np.random.random(g.shape) - 0.5) * 0.01 print('gradient norm', tot ** 0.5) sess.run(momentum, feed_dict = grad_feed_dict) print('gradient_descent', time.time() - tt) # log1 = tf.Print(W1, [W1], 'W1:') # log2 = tf.Print(b1, [b1], 'b1:') # sess.run(log1) # sess.run(log2) mean, norm = memo.stepwise_loss mean /= steps norm /= steps model.update_bn(mean, norm, sess) # if i % 10 == 0: if tot ** 0.5 > 100:# i % 10 == 0: embed() tt = time.time() sim.visualize(memo, show = True, interval = 1, export = exp) sim.visualize(memo, show = False, interval = 1, export = exp) print('visualize', time.time() - tt) if memo.loss < 1e-3: break print('Iter {:5d} time {:.3f} loss {}'.format( i, time.time() - t, memo.loss)) print(i, memo.loss, file = flog) flog.flush()