def addBatch(self, ID, course, faculty, students): batch = Batch() batch.setID(ID) batch.setCourse(course) batch.setFaculty(faculty) batch.setStudents(students) self.batches.append(batch)
def retrieve_citations(self): if not os.path.exists(self.tmp_dir): os.mkdir(self.tmp_dir) #Creates arXiv_citationqueue.txt if it doesn't exist by finding all the gz files in the extract folder if not os.path.exists(self.citation_queue): call('find {source_dir}*.gz -type f > {target_file}'.format( source_dir=self.extract_dir, target_file=self.citation_queue), shell=True) # Initialise some variables batcher = Batch.Batch() while True: file_name = fq.get(self.citation_queue) if file_name is None: break arxiv_id = os.path.splitext(os.path.split(file_name)[1])[0] print "Retrieving citations", arxiv_id uncompressed_tmp = self.tmp_dir + arxiv_id if not os.path.exists(uncompressed_tmp): os.mkdir(uncompressed_tmp) returncode = call( ["tar", "xzf", file_name, "-C", uncompressed_tmp]) if ( returncode == 1 ): #there was an error, so perhaps its not a Tar file. Instead try to decompress with plain old gunzip print "trying to gunzip instead for " + file_name os.system("gunzip -c %s > %s" % (file_name, uncompressed_tmp + "/file.tex")) #Now process .tex files for tex_file_name in os.listdir(uncompressed_tmp): if not (tex_file_name.endswith('.tex') or tex_file_name.endswith('.bbl')): continue citations = self.settings["metadata_reader"].process( arxiv_id, uncompressed_tmp + '/' + tex_file_name) #Store the citations in BibServer self.store_citations(batcher, arxiv_id, citations) #print "CITATIONS for " + arxiv_id #print citations # Delete temporary files if call('rm -R ' + uncompressed_tmp + '*', shell=True): break fq.pop(self.citation_queue) batcher.clear()
def __init__(self, _modelRunTitle, run_codes, _db): # add run title. self.modelRunTitle = self._addTitle(_modelRunTitle) # add run codes. self.run_codes = run_codes # container to pass info around. self.cont = Container.Container() self.cont.set('modelRunTitle', self._addTitle(_modelRunTitle)) self.cont.set('run_codes', run_codes) self.cont.set('path', 'C:/Nonroad/%s/' % (_modelRunTitle)) self.cont.set('db', _db) self.cont.set('qr', qr.QueryRecorder(self.cont.get('path'))) # create Batch runner. self.batch = Batch.Batch(self.cont)
def addStudentToBatch(self, student): listCourseStudent = student.getCouses() for i in range(0, listCourseStudent.__len__()): batchlist = self.getAllBatch() noBatch = 0 for j in range(0, batchlist.__len__()): if (listCourseStudent[i] == batchlist[j].courseName): batchlist[j].addStudent(student) noBatch = 1 if (noBatch == 0): newBatch = b.Batch() newBatch.courseName = listCourseStudent[i] self.listBatch.append(newBatch) i -= 1
def __init__(self, filename, settings=Config.importer['load']['pubmedcentral'], options=[]): self.settings = settings # relevant configuration settings self.options = options # command-line options/arguments self.filename = filename self.procid = uuid.uuid4().hex if self.settings['skip_tar']: self.workdir = self.settings['workdir'] + os.listdir( self.settings['workdir'])[0] + '/' else: self.workdir = self.settings['workdir'] + self.procid + '/' self.b = Batch.Batch() self.m = MetadataReaders.MetadataReaderPMC() if not os.path.exists(self.settings['workdir']): try: os.makedirs(self.settings['workdir']) except: pass if not os.path.exists(self.workdir): os.makedirs(self.workdir)
def post_render_actions(node, actions, queue='3d'): # Proceed with post-render actions (debug, mp4, etc): # Debug images: post_renders = [] if node.parm("debug_images").eval(): for action in actions: # Valid only for Mantra renders; if not isinstance(action, MantraFarm): continue # Generate report per file: debug_render = BatchFarm(job_name = action.parms['job_name'] + "_debug", queue = queue) debug_render.debug_image(action.parms['output_picture'], start = action.parms['start_frame'], end = action.parms['end_frame']) debug_render.node = action.node debug_render.insert_input(action) post_renders.append(debug_render) # Merge reports: merger = BatchFarm(job_name = action.parms['job_name'] + "_mergeReports", queue = queue) ifd_path = os.path.join(os.getenv("JOB"), 'render/sungrid/ifd') merger.merge_reports(action.parms['output_picture'], ifd_path=ifd_path, resend_frames=node.parm('rerun_bad_frames').eval()) merger.node = action.node merger.add_input(debug_render) post_renders.append(merger) # Make a movie from proxy frames: if node.parm("make_proxy").eval() and node.parm("make_movie").eval(): for action in actions: # Valid only for Mantra renders: if not isinstance(action, MantraFarm): continue movie = Batch.BatchFarm(job_name = action.parms['job_name'] + "_mp4", queue = queue) movie.make_movie(action.parms['output_picture']) movie.node = action.node movie.insert_input(action) post_renders.append(movie) return post_renders
def mantra_render_with_tiles(action): '''Creates a series of Mantra jobs using the same ifds stream with different crop setting and overwritten filename. Secondly generates merge job with general BatchFarm class for joining tiles.''' tile_job_ids = [] mantra_tiles = [] tiles_x = action.rop.parm('vm_tile_count_x').eval() tiles_y = action.rop.parm('vm_tile_count_y').eval() parent_job_name = action.parms['job_name'] for tile in range(tiles_x*tiles_y): mantra_farm = MantraFarm(action.node, action.rop, job_name = parent_job_name, \ crop_parms = (tiles_x,tiles_y,tile)) mantra_tiles.append(mantra_farm) # Tile merging job: merging_job_name = action.parms['job_name'] + '_merge' merger = Batch.BatchFarm(job_name = merging_job_name, queue = str(action.node.parm('queue').eval())) merger.node = action.node # NOTE: Only for debugging purposes, we don't rely on this overwise # Need to copy it here, as proxies can be made after tiles merging of course... merger.parms['make_proxy'] = bool(action.node.parm("make_proxy").eval()) # Queue control merger.parms['output_picture'] = mantra_farm.parms['output_picture'] # This is for house keeping only # This prepares commandline to execute: merger.join_tiles(mantra_farm.parms['output_picture'], mantra_farm.parms['start_frame'], mantra_farm.parms['end_frame'], tiles_x*tiles_y) action.insert_outputs(mantra_tiles) [tile.insert_output(merger) for tile in mantra_tiles] # Returns tiles job and merging job. return mantra_tiles, merger
loss_record[mode][k] = loss_record_loaded[mode][k] else: loss_record = {} loss_record['train'] = Utils.Loss_Record() loss_record['val'] = Utils.Loss_Record() rate_counter = Utils.Rate_Counter() DD = Data.Data() timer = {} timer['train'] = Timer(60 * 30) timer['val'] = Timer(60 * 3) trial_loss_record = {} batch = Batch.Batch({'net': net, 'batch_size': P.BATCH_SIZE}) while True: for mode in ['train', 'val']: timer[mode].reset() while not timer[mode].check(): if mode == 'val': net.eval() else: net.train() batch['fill']({'Data': DD, 'mode': mode})
#non-options arguments (lexicon files) argparser.add_argument('files', metavar='FILE', action='store', nargs='*', default=argparse.SUPPRESS, help='dictonary files to learn names from') #Merge configuration files' arguments and cli-given arguments arguments = Selector.loadDefaultArgumentTree() cli_arguments = argparsedToArgument(argparser.parse_args()) arguments.update(cli_arguments) #Custom tweaking of the arguments based on some CLI values if cli_arguments.contains("number"): arguments.set(True, 'gui', '<autogenerate_at_start>') #Set seed if required seedValue = arguments.get('seed', default='auto') if type(seedValue) == int: import random random.seed(int(seedValue)) #Choose to use GUI or Batch if arguments.get('gui', 'enabled', default=False): import GUI GUI.process(arguments) else: import Batch Batch.process(arguments)
def main(): logging.basicConfig(filename='training.log', level=logging.DEBUG) logging.debug(ARGS) # Log arguments net = SqueezeNet() if ARGS.resume_path is not None: cprint('Resuming w/ ' + ARGS.resume_path, 'yellow') net.model_init(ARGS.resume_path) else: net.model_init() net.net.summary() data = Data.Data() batch = Batch.Batch(net) # Maitains a list of all inputs to the network, and the loss and outputs for # each of these runs. This can be used to sort the data by highest loss and # visualize, to do so run: # display_sort_trial_loss(data_moment_loss_record , data) data_moment_loss_record = {} rate_counter = Utils.RateCounter() def run_net(data_index, mode): batch.fill(data, data_index) # Get batches ready batch.forward_backward(data_moment_loss_record, mode) try: epoch = 0 avg_train_loss = Utils.LossLog() avg_val_loss = Utils.LossLog() while True: logging.debug('Starting training epoch #{}'.format(epoch)) # Train mode epoch_train_loss = Utils.LossLog() print_counter = Utils.MomentCounter(ARGS.print_moments) while not data.train_index.epoch_complete: # Epoch of training run_net(data.train_index, 'train') # Run network, Backpropagate # Logging Loss epoch_train_loss.add(data.train_index.ctr, batch.loss) rate_counter.step() if print_counter.step(data.train_index): epoch_train_loss.export_csv( 'logs/epoch%02d_train_loss.csv' % (epoch, )) print('mode = train\n' 'ctr = {}\n' 'most recent loss = {}\n' 'epoch progress = {} \n' 'epoch = {}\n'.format( data.train_index.ctr, batch.loss, 100. * data.train_index.ctr / len(data.train_index.valid_data_moments), epoch)) print('Save model snapshot...') weights_file_name = "epoch{}_save".format(epoch) Utils.save_net(weights_file_name, net, snap=True) K.clear_session() net.model_init( os.path.join(ARGS.save_path, weights_file_name + '_snap.hdf5')) if ARGS.display: batch.display() plt.figure('loss') plt.clf() # clears figure data.train_index.epoch_complete = False logging.info('Avg Train Loss = {}'.format( epoch_train_loss.average())) avg_train_loss.add(epoch, epoch_train_loss.average()) avg_train_loss.export_csv('logs/avg_train_loss.csv') logging.debug('Finished training epoch #{}'.format(epoch)) # Evaluate mode epoch_val_loss = Utils.LossLog() logging.debug('Starting validation epoch #{}'.format(epoch)) print_counter = Utils.MomentCounter(ARGS.print_moments) while not data.val_index.epoch_complete: run_net(data.val_index, 'eval') # Run network epoch_val_loss.add(data.train_index.ctr, batch.loss) if print_counter.step(data.val_index): epoch_val_loss.export_csv('logs/epoch%02d_val_loss.csv' % (epoch, )) print('mode = validation\n' 'ctr = {}\n' 'average val loss = {}\n' 'epoch progress = {} %\n' 'epoch = {}\n'.format( data.val_index.ctr, epoch_val_loss.average(), 100. * data.val_index.ctr / len(data.val_index.valid_data_moments), epoch)) data.val_index.epoch_complete = False avg_val_loss.add(epoch, epoch_val_loss.average()) avg_val_loss.export_csv('logs/avg_val_loss.csv') logging.debug('Finished validation epoch #{}'.format(epoch)) logging.info('Avg Val Loss = {}'.format(epoch_val_loss.average())) Utils.save_net( "epoch%02d_save_%f" % (epoch, epoch_val_loss.average()), net) epoch += 1 except Exception: traceback.print_exc(file=sys.stdout) logging.error(traceback.format_exc()) # Log exception # Interrupt Saves Utils.save_net('interrupt_save', net) epoch_train_loss.export_csv('logs/interrupt%02d_train_loss.csv' % (epoch, )) epoch_val_loss.export_csv('logs/interrupt%02d_val_loss.csv' % (epoch, ))
def run(self): # Check that ElasticSearch is alive self.check_index() # If the user specified the --REBUILD flag, recreate the index if self.options['rebuild']: self.rebuild_index() # Connect to the repository registry = MetadataRegistry() registry.registerReader(self.settings["metadata_format"], self.settings["metadata_reader"]) client = Client(self.settings["uri"], registry) identity = client.identify() print "Connected to repository: %s" % identity.repositoryName() # got to update granularity or we barf with: # oaipmh.error.BadArgumentError: Max granularity is YYYY-MM-DD:2003-04-10T00:00:00Z client.updateGranularity() # Initialise some variables batcher = Batch.Batch() total_records = 0 start = time.time() # Now do the synchonisation # If the user specified an identifier, then synchronise this record if (self.options['identifier'] is not None): total_records += self.synchronise_record( client, batcher, self.options['identifier']) else: # Else, synchronise using the date-range provided by the user, or failing that, # the date-range based on the last sync # Get the synchronisation config record synchronisation_config = self.get_synchronisation_config() if self.options["from_date"] is not None: # If the user specified a from-date argument, use it from_date = self.options[ "from_date"] # already a date (not a datetime) elif synchronisation_config is not None and "to_date" in synchronisation_config: # Else read the last synchronised to_date from the config, and add on a day from_date = dateutil.parser.parse( synchronisation_config["to_date"]).date() + timedelta( days=1) else: # Else use the default_from_date in the config from_date = dateutil.parser.parse( self.settings['default_from_date']).date() if self.options["to_date"] is not None: to_date = self.options[ "to_date"] # already a date (not a datetime) else: to_date = (date.today() - timedelta(days=1)) # Force the from_date to use time 00:00:00 from_date = datetime.combine( from_date, _time(hour=0, minute=0, second=0, microsecond=0)) # Force the to_date to use time 23:59:59 to_date = datetime.combine( to_date, _time(hour=23, minute=59, second=59, microsecond=0)) print "Synchronising from %s - %s" % (from_date, to_date) while from_date < to_date: next_date = datetime.combine( from_date.date() + timedelta(days=(self.settings['delta_days'] - 1)), _time(hour=23, minute=59, second=59, microsecond=0)) number_of_records = self.synchronise_period( client, batcher, from_date, next_date) batcher.clear() #Store the records in elasticsearch self.put_synchronisation_config(from_date, next_date, number_of_records) from_date += timedelta(days=(self.settings['delta_days'])) total_records += number_of_records # Pause so as not to get banned. to = 20 print "Sleeping for %i seconds so as not to get banned." % to time.sleep(to) # Store the records in the index batcher.clear() # Print out some statistics time_spent = time.time() - start print 'Total time spent: %d seconds' % (time_spent) if time_spent > 0.001: # careful as its not an integer print 'Total records synchronised: %i records (%d records/second)' % ( total_records, (total_records / time_spent)) else: print 'Total records synchronised: %i records' % (total_records) return total_records sys.exit()
def train_and_model(model): # TODO LOOK ME this make show all tensor belong cpu or gpu # with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess, tf.device("/CPU:0"): with tf.Session() as sess: # checkpoint val saver = tf.train.Saver() # tensorboard train_writer = tf.summary.FileWriter(FLAGS.dir_train_tensorboard, sess.graph) test_writer = tf.summary.FileWriter(FLAGS.dir_test_tensorboard) # train step print("Train Start...") train_batch_config = Batch.Config() train_batch = Batch.Batch(train_batch_config) sess.run(model["init_op"]) for step in range(FLAGS.max_train_step + 1): key_list = [ Batch.INPUT_DATA, Batch.OUTPUT_LABEL, Batch.OUTPUT_DATA ] data = train_batch.next_batch(FLAGS.batch_size, key_list) feed_dict = { model["X"]: data[Batch.INPUT_DATA], model["Y"]: data[Batch.OUTPUT_DATA], model["Y_label"]: data[Batch.OUTPUT_LABEL] } sess.run(model["train_op"], feed_dict) # print log if step % FLAGS.print_log_step_size == 0: summary_train, _acc, _cost = sess.run( [model["summary"], model["batch_acc"], model["cost"]], feed_dict=feed_dict) print(datetime.datetime.utcnow(), "train step: %d" % step, "batch_acc:", _acc, "cost:", _cost) # checkpoint if step % FLAGS.checkpoint_step_size == 0: saver.save(sess, FLAGS.dir_train_checkpoint, global_step=step) # summary tensorboard if step % FLAGS.summary_step_size: train_writer.add_summary(summary=summary_train, global_step=step) # test step print("Test Start...") test_batch_config = Batch.Config() test_batch = Batch.Batch(test_batch_config) total_acc = 0. for step in range(FLAGS.max_test_step + 1): key_list = [ Batch.INPUT_DATA, Batch.OUTPUT_LABEL, Batch.OUTPUT_DATA ] data = test_batch.next_batch(FLAGS.batch_size, key_list) feed_dict = { model["X"]: data[Batch.INPUT_DATA], model["Y"]: data[Batch.OUTPUT_DATA], model["Y_label"]: data[Batch.OUTPUT_LABEL] } # print("input:", data[Batch.INPUT_DATA]) # print("output:", data[Batch.OUTPUT_DATA]) # print("label:", data[Batch.OUTPUT_LABEL]) summary_test, _acc = sess.run( [model["summary"], model["batch_acc"]], feed_dict=feed_dict) print(datetime.datetime.utcnow(), "test step: %d" % step, "batch_acc: ", _acc) total_acc += _acc if step % FLAGS.print_log_step_size == 0: summary_test, _acc = sess.run( [model["summary"], model["batch_acc"]], feed_dict=feed_dict) # print(datetime.datetime.utcnow(), "test step: %d" % step # , "batch_acc: ", _acc) test_writer.add_summary(summary=summary_test, global_step=step) print("test complete: total acc =", total_acc / (FLAGS.max_test_step + 1)) return
def main(): logging.basicConfig(filename='training.log', level=logging.DEBUG) logging.debug(ARGS) # Log arguments # Set Up PyTorch Environment # torch.set_default_tensor_type('torch.FloatTensor') print ARGS.no_gpu if not ARGS.no_gpu: torch.cuda.set_device(ARGS.gpu) torch.cuda.device(ARGS.gpu) if not ARGS.no_gpu: net = SqueezeNet().cuda() else: net = SqueezeNet() criterion = torch.nn.MSELoss().cuda() optimizer = torch.optim.Adadelta(net.parameters()) if ARGS.resume_path is not None: cprint('Resuming w/ ' + ARGS.resume_path, 'yellow') save_data = torch.load(ARGS.resume_path) net.load_state_dict(save_data) epoch = 0 data = None batch = Batch.Batch(net) if ARGS.bkup is not None: save_data = torch.load(ARGS.bkup) net.load_state_dict(save_data['net']) data = save_data['data'] data.get_segment_data() epoch = save_data['epoch'] else: data = Data.Data() # Maitains a list of all inputs to the network, and the loss and outputs for # each of these runs. This can be used to sort the data by highest loss and # visualize, to do so run: # display_sort_trial_loss(data_moment_loss_record , data) data_moment_loss_record = {} rate_counter = Utils.RateCounter() def run_net(data_index): batch.fill(data, data_index) # Get batches ready batch.forward(optimizer, criterion, data_moment_loss_record) try: backup1 = True avg_val_loss = Utils.LossLog() while True: logging.debug('Starting training epoch #{}'.format(epoch)) net.train() # Train mode print_counter = Utils.MomentCounter(ARGS.print_moments) save_counter = Utils.MomentCounter(ARGS.save_moments) while not data.train_index.epoch_complete: # Epoch of training run_net(data.train_index) # Run network batch.backward(optimizer) # Backpropagate # Logging Loss rate_counter.step() if save_counter.step(data.train_index): save_state = {'data' : data, 'net' : net.state_dict(), 'epoch' : epoch} if backup1: torch.save(save_state, 'backup1.bkup') backup1 = False else: torch.save(save_state, 'backup2.bkup') backup1 = True if print_counter.step(data.train_index): print('mode = train\n' 'ctr = {}\n' 'most recent loss = {}\n' 'epoch progress = {} \n' 'epoch = {}\n' .format(data.train_index.ctr, batch.loss.data[0], 100. * data.train_index.ctr / len(data.train_index.valid_data_moments), epoch)) if ARGS.display: batch.display() plt.figure('loss') plt.clf() # clears figure print_timer.reset() data.train_index.epoch_complete = False logging.debug('Finished training epoch #{}'.format(epoch)) logging.debug('Starting validation epoch #{}'.format(epoch)) epoch_val_loss = Utils.LossLog() print_counter = Utils.MomentCounter(ARGS.print_moments) net.eval() # Evaluate mode while not data.val_index.epoch_complete: run_net(data.val_index) # Run network epoch_val_loss.add(data.train_index.ctr, batch.loss.data[0]) if print_counter.step(data.val_index): epoch_val_loss.export_csv( 'logs/epoch%02d_val_loss.csv' % (epoch,)) print('mode = validation\n' 'ctr = {}\n' 'average val loss = {}\n' 'epoch progress = {} %\n' 'epoch = {}\n' .format(data.val_index.ctr, epoch_val_loss.average(), 100. * data.val_index.ctr / len(data.val_index.valid_data_moments), epoch)) data.val_index.epoch_complete = False avg_val_loss.add(epoch, epoch_val_loss.average()) avg_val_loss.export_csv('logs/avg_val_loss.csv') logging.debug('Finished validation epoch #{}'.format(epoch)) logging.info('Avg Val Loss = {}'.format(epoch_val_loss.average())) Utils.save_net( "epoch%02d_save_%f" % (epoch, epoch_val_loss.average()), net) epoch += 1 except Exception: logging.error(traceback.format_exc()) # Log exception # Interrupt Saves Utils.save_net('interrupt_save', net)
loss_record['train'] = Utils.Loss_Record() loss_record['val'] = Utils.Loss_Record() rate_counter = Utils.Rate_Counter() data = Data.Data() timer = {} timer['train'] = Timer(60*30) timer['val'] = Timer(60*3) print_timer = Timer(args.print_time) save_timer = Timer(args.save_time) trial_loss_record = {} batch = Batch.Batch(net) while True: for mode, data_index in [('train', data.train_index), ('val', data.val_index)]: timer[mode].reset() while not timer[mode].check(): batch.fill(data, data_index) # Get batches ready batch.forward(optimizer, criterion, trial_loss_record) # Run net if mode == 'train': # Backpropagate batch.backward(optimizer) loss_record[mode].add(batch.loss.data[0]) rate_counter.step()
import os, sys import tensorflow as tf import cv2 import gc from tqdm import tqdm_notebook import time import Batch as batch import EDM_Model as edm import vgg.vgg19 as vgg data = batch.Batch(Style_path="./Train_Example/Style", Content_path="./Train_Example/Content") #Opt learning_rate = 0.001 lambda_c = 1. lambda_s = 5. lambda_tv = 1e-5 mini_batch_size = 4 n_epoch = 1 test_check = 200 a_loss = 0 test_loss = 0 #Tensor Flow GPU Memory config = tf.ConfigProto() config.gpu_options.allocator_type = 'BFC' config.gpu_options.allow_growth = True tf.reset_default_graph()
parser.add_option('-s', '--start', dest='start', metavar='S', help='Game to start with (inclusive)', default=1) parser.add_option('-e', '--end', dest='end', metavar='E', help='Game to process up to (inclusive)', default=None) (options, args) = parser.parse_args() logfolder = args[0] if not logfolder[-1] == "/": logfolder += "/" logfolder += "%s.log" start = options.start if start: start = int(start) end = options.end if end: end = int(end) min_ships = lambda id, pw: 100 botid = 1 bots = [] for i in range(50): bots.append({'type': VariableAggressionPlayer, 'params': {'conservativeness': i/50., 'id': botid}}) botid += 1 subjects = [{'type': Dave2Player, 'params': {'id': botid}}] maps = [] for i in range(100): maps.append(file('../../newmaps/map%d.txt' % (i + 1)).read()) Batch.batch_challenge(subjects, bots, maps, logfolder, start, end)
help="""Presets of filters to apply on generated names.""" ) #non-options arguments (lexicon files) argparser.add_argument( 'files', metavar='FILE', action='store', nargs='*', default=argparse.SUPPRESS, help='dictonary files to learn names from' ) #Merge configuration files' arguments and cli-given arguments arguments = Selector.loadDefaultArgumentTree() cli_arguments = argparsedToArgument( argparser.parse_args() ) arguments.update( cli_arguments ) #Custom tweaking of the arguments based on some CLI values if cli_arguments.contains("number"): arguments.set( True, 'gui', '<autogenerate_at_start>' ) #Set seed if required seedValue = arguments.get('seed',default='auto') if type(seedValue) == int: import random random.seed(int(seedValue)) #Choose to use GUI or Batch if arguments.get('gui','enabled',default=False): import GUI GUI.process( arguments ) else: import Batch Batch.process( arguments )
#text_field = data.Field(lower=True) #label_field = data.Field(sequential=False) # train_iter, dev_iter = mr(text_field, label_field, device=-1, repeat=False) #train_iter, dev_iter, test_iter = sst(text_field, label_field, device=-1, repeat=False, shuffle=True, sort=False) file = Dictionary.LoadData() file.form_data() word = Dictionary.Word(data=file.all_file, fine_tune=True) word.build_dict() table = Dictionary.WordTable(vocab=word.vocab) table.build_table() args.itos = table.itos batch_iter = Batch.Iterator(batch_len=(50, args.batch_size, len(file.all_file[2]['data'])), data=file.all_file, vocab=table) data_iter, label_iter = batch_iter.create_iter() test_iter, train_iter, dev_iter = zip(data_iter, label_iter) # update args and print args.embed_num = word.vocab_num + 2 args.class_num = word.label_num args.cuda = (not args.no_cuda) and torch.cuda.is_available() del args.no_cuda args.kernel_sizes = [int(k) for k in args.kernel_sizes.split(',')] args.save_dir = os.path.join( args.save_dir, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) if not os.path.isdir(args.save_dir): os.makedirs(args.save_dir)
logfolder += "%s.log" start = options.start if start: start = int(start) end = options.end if end: end = int(end) min_ships = lambda id, pw: 100 botid = 1 bots = [] for i in range(50): bots.append({ 'type': VariableAggressionPlayer, 'params': { 'conservativeness': i / 50., 'id': botid } }) botid += 1 subjects = [{'type': Dave2Player, 'params': {'id': botid}}] maps = [] for i in range(100): maps.append(file('../../newmaps/map%d.txt' % (i + 1)).read()) Batch.batch_challenge(subjects, bots, maps, logfolder, start, end)
import sys, action, status, helpers, optimizeBatch import Batch # new imports start here # settings = helpers.get_settings() try: action = str(sys.argv[1]) except: action = None try: PATH = str(sys.argv[2]) except: PATH = None if action == 'status' or action == None: status.execute() elif action == '-o': optimizeBatch.execute(PATH) elif action == "batch": Batch.execute() # new actions start here
import sys sys.path.append('C:\\Users\\Michael\\Documents\\My Dropbox\\Projects\\Python Workspace\\PlanetWars\\src') import Batch import logging from Players import * from PlanetWarsProxy import PlanetWarsProxy if __name__ == "__main__": min_ships = lambda id, pw: 100 bots = [] for i in range(100): bots.append(VariableAggressionPlayer(i/100., min_ships)) maps = [file('../newmaps/map1.txt').read()] logging.basicConfig(filename='../experiments/config-3/map-1/results.log', level=logging.INFO) Batch.batch_run(bots, maps)
loss_record['train'] = Utils.Loss_Record() loss_record['val'] = Utils.Loss_Record() rate_counter = Utils.Rate_Counter() data = Data.Data() timer = {} timer['train'] = Timer(args.mini_train_time) timer['val'] = Timer(args.mini_val_time) print_timer = Timer(args.print_time) save_timer = Timer(args.save_time) data_moment_loss_record = {} batch = Batch.Batch(net) while True: for mode, data_index in [('train', data.train_index), ('val', data.val_index)]: timer[mode].reset() while not timer[mode].check(): batch.fill(data, data_index) # Get batches ready batch.forward(optimizer, criterion, data_moment_loss_record) # Run net, forward pass if mode == 'train': # Backpropagate batch.backward(optimizer) loss_record[mode].add(batch.loss.data[0]) rate_counter.step()
import sys sys.path.append( 'C:\\Users\\Michael\\Documents\\My Dropbox\\Projects\\Python Workspace\\PlanetWars\\src' ) import Batch import logging from Players import * from PlanetWarsProxy import PlanetWarsProxy if __name__ == "__main__": min_ships = lambda id, pw: 100 bots = [] for i in range(100): bots.append(VariableAggressionPlayer(i / 100., min_ships)) maps = [file('../newmaps/map1.txt').read()] logging.basicConfig(filename='../experiments/config-3/map-1/results.log', level=logging.INFO) Batch.batch_run(bots, maps)
if __name__ == "__main__": parser = optparse.OptionParser(usage="Usage: %prog [options] logdir", description=DESC) parser.add_option('-s', '--start', dest='start', metavar='S', help='Game to start with (inclusive)', default=1) parser.add_option('-e', '--end', dest='end', metavar='E', help='Game to process up to (inclusive)', default=None) (options, args) = parser.parse_args() logfolder = args[0] if not logfolder[-1] == "/": logfolder += "/" logfolder += "%s.log" start = options.start if start: start = int(start) end = options.end if end: end = int(end) min_ships = lambda id, pw: 100 botid = 1 bots = [] for i in [True, False]: bots.append({'type': PredictingPlayer, 'params': {'scout_enabled': i, 'id': botid}}) botid += 1 maps = [] for i in range(100): maps.append(file('../../newmaps/map%d.txt' % (i + 1)).read()) Batch.batch_run(bots, maps, logfolder, start, end)