def run(self): while True: urls = self.check(interval=5 * 60) filename = utils.generate_filename(self.room_id) c_filename = os.path.join(os.getcwd(), 'files', filename) self.record(urls[0], c_filename) self.print(self.room_id, '录制完成')
def run(self) -> None: logging.basicConfig( level=utils.get_log_level(self.config), format= '%(asctime)s %(thread)d %(threadName)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S', handlers=[ logging.FileHandler(os.path.join( self.config['root']['logger']['log_path'], "LiveRecoder_" + datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + '.log'), "a", encoding="utf-8") ]) while True: try: if self.live_status: urls = self.get_live_urls() filename = utils.generate_filename(self.room_id) c_filename = os.path.join(self.record_dir, filename) self.record(urls[0], c_filename) logging.info(self.generate_log('录制完成' + c_filename)) else: logging.info(self.generate_log('下播了')) break except Exception as e: logging.error( self.generate_log('Error while checking or recording:' + str(e) + traceback.format_exc()))
def db_test(): db = get_db() db.execute( "INSERT INTO IMAGE (NAME, LONGITUDE, LATITUDE) VALUES (?, ?, ?)", [generate_filename("test.jpg"), 15, 25]) db.commit() return 'Insert Successful'
def save(self): buff = self.get_buffer() text = buff.get_text(buff.get_start_iter(), buff.get_end_iter()) if not self.note: import utils self.note = Note(utils.generate_filename(), utils.user_configurations['TIXPATH'], text) self.note.write_text_to_file(text) return self.note
def run(self): while True: try: self.print(self.room_id, '等待开播') urls = self.check(interval=self.check_interval) filename = utils.generate_filename(self.room_id) self.record(urls, filename) self.print(self.room_id, '录制完成' + filename) except Exception as e: self.print(self.room_id, 'Error while checking or recording:' + str(e))
def write_config(self): with open( os.path.join("sqbs_configs", generate_filename(self.email, ".json")), 'w') as f: json.dump( { "agg_id": self.aggregate_id, "roster_id": self.roster_id, "last_run": 0 }, f)
def save(self): data = self.cleaned_data filename = generate_filename(data['image'].name) handler = ImageHandlerInstrument() handler.load(filename, data['image']) handler.save_thumbnails('PNG') instrument = Instrument.objects.create( name=data['name'], description=data.get('description'), image=filename ) self.instance = instrument
def run(self): while True: try: urls = self.check(interval=self.check_interval) filename = utils.generate_filename(self.room_id) utils.checkRecordDirExisted() c_filename = os.path.join(os.getcwd(), 'files', filename) self.record(urls[0], c_filename) self.print(self.room_id, '录制完成' + c_filename) except Exception as e: self.print(self.room_id, 'Error while checking or recording:' + str(e))
def snap(self): """ Snap an image and saves it to /static/images/image-**timestamp**.jpg """ filename = generate_filename("jpg") with picamera.Picamera() as camera: camera.start_preview() time.sleep(2) camera.capture(filename) camera.stop_preview() return filename
def save(self): data = self.cleaned_data filename = generate_filename(data['file'].name) handler = FileHandlerDocument() handler.load(filename, data['file']) document = DocumentPlayerRepertoryItem(name=data['file'].name, document=filename, player_repertory_item_id=data['player_repertory_item']) if handler.is_image(): handler.save_thumbnails('PNG') document.type = DocumentType.image document.save() return document
def add_content(key): key_folder = os.path.join(config.data_folder, key) if not os.path.exists(key_folder): logging.info("Folder %s does not exists ; creating.", key_folder) os.mkdir(key_folder) target_name = generate_filename(key) with open(os.path.join(key_folder, target_name), "w") as f: logging.info("User identified by key '%s' has pushed a file.", key) f.write(request.data.decode()) return "/%s" % target_name
def shoot(self): """ Shoots a video and saves it to /static/videos/video-**timestamp**.h264 """ filename = generate_filename('h264', 'videos') with picamera.Picamera() as camera: camera.resolution = (640, 480) camera.start_preview() camera.start_recording(filename) camera.wait_recording(10) camera.stop_recording() camera.stop_preview() return filename
def run(self): while True: try: urls = self.check(interval=self.check_interval) filename = utils.generate_filename() utils.checkRecordDirExisted() if not os.path.exists(os.path.join(self.out_dir, self.room_owner)): os.makedirs(os.path.join(self.out_dir, self.room_owner)) c_filename = os.path.join(self.out_dir, self.room_owner, filename) self.record(urls[0], c_filename) self.print(self.room_id, '录制完成' + c_filename) except Exception as e: self.print(self.room_id, 'Error while checking or recording:' + str(e))
def validate_convert_args(args): err_dict = { "missing": "Missing {}.", "unauthorized": "Your email is invalid or unauthorized! Make sure this is the same email you used to create your scoresheet.", "rounds_min": "First round number invalid - it must be a number no greater than {}.". format(MAX_ROUNDS), "rounds_max": "Last round number invalid - it must be a number no greater than {}.". format(MAX_ROUNDS), "min_lt_max": "First round number can't be greater than last round number." } # check if any required arguments aren't present (after client-side validation) for check_var in ("email", "rounds_min", "rounds_max"): if check_var not in args: return {"error": err_dict["missing"].format(check_var)} # validate against pre-approved emails if not authorize_email(args["email"]): return {"error": err_dict["unauthorized"]} # check round min and round max are integers, in the correct range, and min < max try: args["rounds_min"] = int(args["rounds_min"]) assert args["rounds_min"] <= MAX_ROUNDS and args["rounds_min"] > 0 except: return {"error": err_dict["rounds_min"]} try: args["rounds_max"] = int(args["rounds_max"]) assert args["rounds_max"] <= MAX_ROOMS and args["rounds_max"] > 0 except: return {"error": err_dict["rounds_max"]} try: assert args["rounds_min"] <= args["rounds_max"] except: return {"error": err_dict["min_lt_max"]} # if user has created sheets before, there'll be a sqbs config for it if os.path.isfile( os.path.join("sqbs_configs", generate_filename(args["email"], ".json"))): return False return {"error": err_dict["unauthorized"]}
def run(self): status = NOT_START c_filename = None while True: try: urls = self.check(interval=self.check_interval, blocking=False) if urls is None and status == NOT_START: # just wait self.print(self.room_id, '等待开播') time.sleep(self.check_interval) if urls and status == NOT_START: # start recording status = self.next_status(status, True) filename = utils.generate_filename(self.room_id) c_filename = os.path.join(os.getcwd(), 'files', filename) self.print(self.room_id, '开始录制' + c_filename) self.record(urls[0], c_filename) if urls and status == RECORDING: # recording self.record(urls[0], c_filename) if urls is None and status == RECORDING: # stream end self.print(self.room_id, '录制完成' + c_filename) status = self.next_status(status, True) subprocess.run( 'ffmpeg -y -i {c_filename} -c:a copy -c:v copy {final_name}' .format(c_filename=c_filename, final_name=c_filename.split('.')[0] + '_final.mp4'), shell=True, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) print("ffmpeg done.") try: if callable(self.on_stop): self.on_stop( c_filename.split('.')[0] + '_final.mp4') # callback elif self.on_stop is not None: raise NotcallableError('on_stop is not callable') except Exception as e: self.print( self.room_id, 'Error while calling on_stop callback' + str(e)) except Exception as e: self.print(self.room_id, 'Error while checking or recording:' + str(e)) status = self.next_status(status, False)
def run(self) -> None: while True: try: if self.live_status: urls = self.get_live_urls() filename = utils.generate_filename(self.room_id) c_filename = os.path.join(self.record_dir, filename) self.record(urls[0], c_filename) logging.info(self.generate_log('录制完成' + c_filename)) else: logging.info(self.generate_log('下播了')) break except Exception as e: logging.error( self.generate_log('Error while checking or recording:' + str(e)))
def upload_file(): if request.method == 'POST': # check if the post request has the file part if 'file' not in request.files: flash('No file part') return redirect(request.url) file = request.files['file'] # if user does not select file, browser also # submit an empty part without filename if file.filename == '': flash('No selected file') return redirect(request.url) elif file and allowed_file(file.filename): filename = generate_filename(file.filename) file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename)) return redirect(url_for('uploaded_file', filename=filename)) return '''
def api_generate_questions(): user_id = request.headers.get('User-Id') if user_id is None: return utils.response(403, 'Unauthorized') json_data = request.get_json() if 'content' not in json_data or len(json_data['content']) < 20: return utils.response(400, 'Content is not valid') content = json_data['content'].strip() content_hash = str( hashlib.sha256(content.encode('utf-8')).hexdigest()[:32]).strip() article = mongo.db.user_articles.find_one({'id': content_hash}) if article is not None: questions = list( mongo.db.user_questions.find({'article_id': content_hash})) else: article_id = content_hash article = { 'id': article_id, 'type': 'text', 'created_time': datetime.datetime.utcnow(), 'publish_time': datetime.datetime.utcnow(), 'thumbnail': '', 'title': '', 'content': content } questions = generate_questions(content) for ques in questions: ques['id'] = utils.generate_filename() ques['article_id'] = article_id mongo.db.user_articles.insert_one(article) mongo.db.user_questions.insert_many(questions) num_sent = sum([1 if len(sent) > 20 else 0 for sent in content.split('.')]) max_count = min(num_sent, 10) questions = question_recommender.recommend(questions, user_id, max_count=max_count) questions = filter(questions, ['answer', 'explain']) article['questions'] = list(questions) return utils.response(200, 'Success', article)
def on_save_draft(self, action, paraneter): """ Save post data to the draft file """ dialog = Gtk.FileChooserDialog( "Save draft", self.main_window, Gtk.FileChooserAction.SAVE, (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_SAVE, Gtk.ResponseType.OK)) save_button = dialog.get_header_bar().get_children()[1] save_button.get_style_context().add_class("suggested-action") # Add filters draft_filter = Gtk.FileFilter() draft_filter.set_name("Draft files") draft_filter.add_pattern("*.sbd") dialog.add_filter(draft_filter) any_filter = Gtk.FileFilter() any_filter.set_name("Any files") any_filter.add_pattern("*") dialog.add_filter(any_filter) post_title = self.main_window.title_entry.get_text() dialog.set_current_name(utils.generate_filename(post_title) + u".sbd") dialog.set_current_folder(os.path.expanduser("~")) response = dialog.run() if response == Gtk.ResponseType.OK: filename = dialog.get_filename() with open(filename, "wb") as draft_file: draft_obj = { "title": post_title, "body": self.main_window.sourceview.get_buffer().props.text, "tags": self.main_window.tag_entry.get_text() } pickle.dump(draft_obj, draft_file) self.main_window.infobar.get_content_area().get_children( )[0].set_text("Draft was succesfully saved") self.main_window.infobar.get_action_area().get_children( )[1].props.visible = True self.main_window.infobar.show() dialog.destroy()
def speak_submit(): try: authorId = request.form['authorId'] recordings = [] for item in request.files: if (item not in CLASSES): continue class_path = join(DATASET_DIRECTORY, item) if (not exists(class_path)): mkdir(class_path) file = request.files[item] filename = generate_filename(authorId, 'wav') filepath = join(class_path, filename) file.save(filepath) recordings.append((filepath, authorId, item)) insert_recordings(recordings) except: return ('', 400) return ('', 200)
def on_save_draft(self, action, paraneter): """ Save post data to the draft file """ dialog = Gtk.FileChooserDialog( "Save draft", self.main_window, Gtk.FileChooserAction.SAVE, (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_SAVE, Gtk.ResponseType.OK) ) save_button = dialog.get_header_bar().get_children()[1] save_button.get_style_context().add_class("suggested-action") # Add filters draft_filter = Gtk.FileFilter() draft_filter.set_name("Draft files") draft_filter.add_pattern("*.sbd") dialog.add_filter(draft_filter) any_filter = Gtk.FileFilter() any_filter.set_name("Any files") any_filter.add_pattern("*") dialog.add_filter(any_filter) post_title = self.main_window.title_entry.get_text() dialog.set_current_name(utils.generate_filename(post_title) + u".sbd") dialog.set_current_folder(os.path.expanduser("~")) response = dialog.run() if response == Gtk.ResponseType.OK: filename = dialog.get_filename() with open(filename, "wb") as draft_file: draft_obj = { "title": post_title, "body": self.main_window.sourceview.get_buffer().props.text, "tags": self.main_window.tag_entry.get_text() } pickle.dump(draft_obj, draft_file) self.main_window.infobar.get_content_area().get_children()[0].set_text("Draft was succesfully saved") self.main_window.infobar.get_action_area().get_children()[1].props.visible = True self.main_window.infobar.show() dialog.destroy()
def run(self): try: while True: res = self.check() if isinstance(res, list): self.fname = utils.generate_filename( self.room_id, self.room_info['roomname']) if self.capture_danmaku: self.dmlogger = dmxml.BLiveXMLlogger(self.room_id, uid=0) self.dmlogger.run(saving_path=os.path.join( self.saving_path, self.fname + '.xml')) self.lalogger = dmlar.BLiveLARlogger(self.room_id, uid=0) self.lalogger.run(saving_path=os.path.join( self.saving_path, self.fname + '.lar')) self.stream_rec_thread = threading.Thread( target=self.record, args=(res[0], )) self.stream_rec_thread.start() if res: if self.recording_lock.acquire( timeout=self.check_interval): self.stream_rec_thread.join() if self.capture_danmaku: self.dmlogger.terminate() del self.dmlogger self.lalogger.terminate() del self.lalogger self.recording_lock.release() else: time.sleep(self.check_interval) except Exception as e: utils.print_log(self.room_id, 'Error while checking or recording:' + str(e)) finally: if self.recording_lock.locked() and self.capture_danmaku: self.dmlogger.terminate() del self.dmlogger self.lalogger.terminate() del self.lalogger self.recording_lock.release()
def convert(): req = dict((i, j.strip()) for i, j in zip(request.args.keys(), request.args.values())) invalid = validate_convert_args(req) if invalid: return json.dumps(invalid) filename = generate_filename(req["email"], ".json") full_filename = os.path.join("sqbs_configs", filename) d = {} with open(full_filename) as f: d = json.load(f) if int(time.time()) - d["last_run"] < CONVERSION_REPEAT_DELAY: return json.dumps({ "error": "Please wait at least {} seconds in between submitting sqbs conversion jobs" .format(CONVERSION_REPEAT_DELAY + CONVERSION_SCHEDULE_INTERVAL) }) for item in sqbs_queue: if item[0] == full_filename: return json.dumps({ "error": "You already have a job request for that aggregate sheet" }) if not isinstance(req["rounds_min"], int) or not isinstance( req["rounds_max"], int): return json.dumps({"error": "Round numbers must be integers"}) log.info(f"[{filename}] -- adding to SQBS queue -- {sqbs_queue}") sqbs_queue.append( (full_filename, int(req["rounds_min"]), int(req["rounds_max"]))) with open(full_filename, "w") as f: d["email"] = req["email"] json.dump(d, f) return json.dumps({"success": req["email"]})
def doc_handler(msg): chat_id = msg.chat.id if msg.caption == "/get_song": doc = msg.document filename = doc.file_name file_id = doc.file_id real_filename = utils.generate_filename() if filename.split('.')[-1] != 'osz': bot.send_message(chat_id, "Wrong file extension") return if doc.file_size > 10485760: bot.send_message(chat_id, "File size is too large") return utils.get_file(file_id, f"temp/{real_filename}.zip") check = utils.get_song(real_filename) if check != "ok": bot.send_message(chat_id, "That's more than one song, idk which one you want...") utils.delete_temp_files(zip_filename=f"temp/{real_filename}.zip") return bot.send_document(chat_id, open(f"temp/{real_filename}.mp3", 'rb'), timeout=40) utils.delete_temp_files(zip_filename=f"temp/{real_filename}.zip", mp3_filename=f"temp/{real_filename}.mp3") return
def main(args): # Training settings parser = argparse.ArgumentParser( description='PyTorch Attentive RNN Language Modeling') parser.add_argument('--batch-size', type=int, default=8, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--epochs', type=int, default=50, metavar='N', help='number of epochs to train (default: 40)') parser.add_argument('--lr', type=float, default=30.0, metavar='LR', help='learning rate (default: 30.0)') parser.add_argument('--patience', type=int, default=5, metavar='P', help='patience for lr decrease (default: 5)') parser.add_argument('--seed', type=int, default=123, metavar='S', help='random seed (default: 123)') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help= 'how many batches to wait before logging training status (default 10)') parser.add_argument('--dataset', default='shakespeare', const='shakespeare', nargs='?', choices=['wiki-02', 'ptb', 'shakespeare'], help='Select which dataset (default: %(default)s)') parser.add_argument( '--embedding-size', type=int, default=150, metavar='N', help='embedding size for embedding layer (default: 20)') parser.add_argument('--n-layers', type=int, default=5, metavar='N', help='layer size for RNN encoder (default: 1)') parser.add_argument('--hidden-size', type=int, default=150, metavar='N', help='hidden size for RNN encoder (default: 20)') parser.add_argument( '--positioning-embedding', type=int, default=150, metavar='N', help='hidden size for positioning generator (default: 20)') parser.add_argument('--input-dropout', type=float, default=0.5, metavar='D', help='input dropout (default: 0.5)') parser.add_argument('--rnn-dropout', type=float, default=0.0, metavar='D', help='rnn dropout (default: 0.0)') parser.add_argument('--decoder-dropout', type=float, default=0.5, metavar='D', help='decoder dropout (default: 0.5)') parser.add_argument( '--clip', type=float, default=0.25, metavar='N', help='value at which to clip the norm of gradients (default: 0.25)') parser.add_argument('--optim', default='sgd', const='sgd', nargs='?', choices=['sgd', 'adam', 'asgd'], help='Select which optimizer (default: %(default)s)') parser.add_argument( '--salton-lr-schedule', help= 'Enables same training schedule as Salton et al. 2017 (default: False)', action='store_true') parser.add_argument('--early-stopping-patience', type=int, default=35, metavar='P', help='early stopping patience (default: 25)') parser.add_argument('--attention', help='Enable standard attention (default: False', action='store_true') parser.add_argument('--no-positional-attention', help='Disable positional attention (default: False', action='store_false') parser.add_argument( '--tie-weights', help='Tie embedding and decoder weights (default: False', action='store_true') parser.add_argument( '--file-name', action="store", help= 'Specific filename to save under (default: uses params to generate', default=False) parser.add_argument('--parallel', help='Enable using GPUs in parallel (default: False', action='store_true') args = parser.parse_args(args) if not args.file_name: args.file_name = generate_filename(args) torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) writer = SummaryWriter('runs/' + args.file_name) train_iter, valid_iter, test_iter, vocab = get_dataset( dataset=args.dataset, batch_size=args.batch_size, device=device) args.vocab_size = len(vocab) model = get_model(args) if torch.cuda.device_count() > 1 and args.parallel: print("Using", torch.cuda.device_count(), "GPUs") model = nn.DataParallel(model) else: if args.parallel: warnings.warn( "Passed in parallel flag but torch unable to detect multiple GPUs" ) args.parallel = False model.to(device) print(str(model)) print(count_parameters(model)) # Training Set Up if args.optim == 'sgd': optimizer = optim.SGD(model.parameters(), lr=args.lr, weight_decay=12e-7) if args.optim == 'asgd': optimizer = optim.ASGD(model.parameters(), lr=args.lr, weight_decay=12e-7) if args.optim == 'adam': optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(0.0, 0.999), eps=1e-8, weight_decay=12e-7, amsgrad=True) criterion = nn.CrossEntropyLoss(ignore_index=0, reduction='sum') scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=args.patience, verbose=True, factor=0.5) early_stopping_counter = 0 best_val_loss = False # At any point you can hit Ctrl + C to break out of training early. try: # Loop over epochs. for epoch in range(1, args.epochs + 1): if args.salton_lr_schedule: current_learning_rate = 0.5**max(epoch - 12, 0.0) optimizer.param_groups[0]['lr'] = current_learning_rate epoch_start_time = time.time() train(args, model, train_iter, valid_iter, criterion, optimizer, epoch, writer) if args.parallel: with open('models/temp.pt'.format(args.file_name), 'wb') as fw: torch.save(model.module.state_dict(), fw) with open('models/temp.pt'.format(args.file_name), 'rb') as fr: single_gpu_model = get_model(args) # load on either single gpu or on the cpu (if gpu not avail) single_gpu_model.load_state_dict( torch.load(fr, map_location=device)) single_gpu_model.to(device) val_loss = evaluate(args, single_gpu_model, valid_iter, criterion, save_attention=True, epoch=epoch, vocabulary=vocab) del single_gpu_model else: val_loss = evaluate(args, model, valid_iter, criterion, save_attention=True, epoch=epoch, vocabulary=vocab) test_loss = evaluate(args, model, test_iter, criterion) # possibly update learning rate scheduler.step(val_loss) # track learning ratei writer.add_scalar('lr', optimizer.param_groups[0]['lr'], epoch) writer.add_scalar('validation_loss_at_epoch', val_loss, epoch) writer.add_scalar('test_loss_at_epoch', test_loss, epoch) writer.add_scalar('validation_perplexity_at_epoch', min(math.exp(min(val_loss, 7)), 1000), epoch) writer.add_scalar('test_perplexity_at_epoch', min(math.exp(min(test_loss, 7)), 1000), epoch) print('-' * 89) print( '| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | ' 'valid ppl {:8.2f}'.format(epoch, (time.time() - epoch_start_time), val_loss, math.exp(min(val_loss, 7)))) print('-' * 89) # Save the model if the validation loss is the best we've seen so far. if not best_val_loss or val_loss < best_val_loss: if not os.path.exists('models'): os.makedirs('models') with open('models/{}.pt'.format(args.file_name), 'wb') as f: if args.parallel: torch.save(model.module.state_dict(), f) else: torch.save(model.state_dict(), f) best_val_loss = val_loss early_stopping_counter = 0 else: early_stopping_counter += 1 writer.add_scalar('best_validation_perplexity_at_epoch', min(math.exp(min(best_val_loss, 7)), 1000), epoch) if early_stopping_counter >= args.early_stopping_patience: print("Validation loss has not improved for {}".format( early_stopping_counter)) print("Ending Training early at epoch {}".format(epoch)) break except KeyboardInterrupt: print('-' * 89) print('Exiting from training early') if os.path.exists('models/{}.pt'.format(args.file_name)): # Load the best saved model. with open('models/{}.pt'.format(args.file_name), 'rb') as f: # instantiate model model = get_model(args) # load on either single gpu or on the cpu (if gpu not avail) model.load_state_dict(torch.load(f, map_location=device)) model.to(device) # after load the rnn params are not a continuous chunk of memory # this makes them a continuous chunk, and will speed up forward pass model.flatten_parameters() # Run on test data. test_loss = evaluate(args, model, test_iter, criterion) print('=' * 89) print( '| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format( test_loss, math.exp(test_loss))) print('=' * 89)
total_loss_sum += info[0] ctr_sum += 1 total_ctr += info[1] if (i + 1) % global_data_print_freq == 0: write_file_and_close(global_output_filename, "epoch: {:d}, " "train set index: {:d}, " "average loss: {:.10f}" .format(epoch, i, running_loss_sum / ctr_sum) ) running_loss_sum = 0.0 ctr_sum = 0 it = it + 1 write_file_and_close(global_output_filename, "Epoch {:d} finished, average loss: {:.10f}" .format(epoch, total_loss_sum / total_ctr) ) if (epoch + 1) % global_epoch_test_freq == 0: write_file_and_close(global_output_filename, "Starting testing") info = [0., 0., 0.] test(info) write_file_and_close(global_output_filename, "Correct: {:d}, total: {:d}, " "accuracy: {:.10f}, average loss: {:.10f}" .format(info[0], info[1], info[0] / info[1], info[2] / info[1]) ) write_file_and_close(global_output_filename, "Finished testing") model_filename = generate_filename() torch.save(net, model_filename)
def run( experiment_class='Experiment', name='', batch_size=100, max_iters=10000, filepath='experiments', random_seed=0, parallel=False, verbose=False, ): print(experiment_class, name, batch_size, max_iters, random_seed) #Task meta_selector = BudgetMetaSelector() env = gym.make('LQG1D-v0') R = np.asscalar(env.Q * env.max_pos**2 + env.R * env.max_action**2) M = env.max_pos gamma = env.gamma H = env.horizon tp = TaskProp(gamma, H, -env.max_action, env.max_action, R, M, -env.max_pos, env.max_pos, 2 * env.max_action) local = True #Policy theta_0 = -0.1 w = math.log(1) #math.log(env.sigma_controller) pol = ExpGaussPolicy(theta_0, w) #Features feature_fun = utils.identity #Constraints constr = OptConstr(delta=0.2, N_min=batch_size, N_max=500000, N_tot=30000000, max_iter=max_iters, approximate_gradients=True) #Evaluation of expected performance def evaluate(pol, deterministic=False): var = 0 if deterministic else pol.cov return env.computeJ(pol.theta_mat, var) #Run # exp = adaptive_exploration.Experiment(env, tp, grad_estimator, meta_selector, constr, feature_fun, evaluate, name=name) # exp = adaptive_exploration.CollectDataExperiment(env, tp, grad_estimator, meta_selector, constr, feature_fun, evaluate, name=name) # exp = adaptive_exploration.SafeExperiment(env, tp, grad_estimator, meta_selector, constr, feature_fun, evaluate, name=name, random_seed=random_seed) experiment = AVAILABLE_EXPERIMENTS[experiment_class] exp = experiment(env, tp, meta_selector, constr, feature_fun, evaluate=evaluate, name=name, random_seed=random_seed) exp.run(pol, local, parallel, verbose=verbose, filename=os.path.join(filepath, name + utils.generate_filename()))
def run(self): while True: try: status, info = self.check(interval=self.check_interval) if not status: self.print(room_id=self.room_id, content='Not Broadcasting...') time.sleep(self.check_interval) continue self.print(room_id=self.room_id, content='Start Broadcasting!') self.inform(text=f"{self.room_id}开播了", desp=info['roomname']) # time.sleep(1.5) # status, urls = self.get_live_urls_byapi() # url = urls[0] # print(urls) # print("url:" + url) # path = urlparse(url).path # ext = str(os.path.splitext(path)[1]) # print(ext) # Directly use get_live_urls_byweb, prevent m3u8sucks time.sleep(1) status, urls = self.get_live_urls_byweb() if not status: self.print("Error! Getting play_url From Web Failed.") self.inform( text= f"Error! Getting play_url From Web Failed. Room:{self.room_id}", desp=info['roomname']) time.sleep(3) continue print(urls) ext = str(os.path.splitext(urlparse(urls[0]).path)[1]) print(ext) # generate file name filename = utils.generate_filename(self.room_id) c_filename = os.path.join(os.getcwd(), 'files', filename) if self.use_option_link: # If you use this tool overseas, you maybe set use_option_link to True and use a proxy. # use the last url in urls, which is generally available for overseas user. url = urls[len(urls) - 1] else: # use the first url in urls, which is generally available for domenstic user. url = urls[0] if ext == '.flv': self.flvrecord(url, c_filename + ".flv") elif ext == '.m3u8': self.print( f"Warning! Recording in m3u8 mode, maybe the record file is not complete." ) self.inform( text= f"Warning! Recording in m3u8 mode, maybe the record file is not complete. Room:{self.room_id}", desp=info['roomname']) self.m3u8record(url, c_filename + ".mp4") self.print(self.room_id, '录制完成' + c_filename) self.inform(text=f"{self.room_id}录制结束", desp="") except Exception as e: self.print(self.room_id, 'Error while checking or recording:' + str(e)) traceback.print_exc() self.inform(text=f"Error!", desp=str(e))
def run( self, policy, use_local_stats=False, # Update task prop only with local stats parallel=True, filename=generate_filename(), verbose=False): self.use_local_stats = True self.initial_configuration = self.get_param_list(locals()) self.estimator = Estimators(self.task_prop, self.constr) N = N_old = self.constr.N_min # Total number of trajectories to take in this iteration N1, N2, N3 = self.split_trajectory_count(N) #Multiprocessing preparation if parallel: self._enable_parallel() # COMPUTE BASELINES features, actions, rewards, prevJ, gradients = self.get_trajectories_data( policy, N, parallel=parallel) prevJ_det = self.estimate_policy_performance(policy, N, parallel=parallel, deterministic=True, get_min=False) #Learning iteration = 0 N_tot = 0 start_time = time.time() J_hat = prevJ J_journey = (2 * prevJ + prevJ_det) / 3 while iteration < self.constr.max_iter: iteration += 1 J_det_exact = self.evaluate(policy, deterministic=True) self.make_checkpoint(locals()) # CHECKPOINT BEFORE SIGMA STEP J_journey = 0 # PRINT if verbose: if iteration % 50 == 1: print( 'IT\tN\t\tJ\t\t\tJ_DET\t\t\tTHETA\t\tSIGMA\t\t\tBUDGET' ) print(iteration, '\t', N, '\t', J_hat, '\t', prevJ_det, '\t', policy.get_theta(), '\t', policy.sigma, '\t', self.budget / N, '\t', time.time() - start_time) start_time = time.time() # PERFORM FIRST STEP features, actions, rewards, J_hat, gradients = self.get_trajectories_data( policy, N1, parallel=parallel) J_journey += J_hat * N1 if iteration > 1: self.budget += N3 * ( J_hat - prevJ) # B += J(theta, sigma') - J(theta, sigma) prevJ = J_hat alpha, N1, safe = self.meta_selector.select_alpha( policy, gradients, self.task_prop, N1, iteration, self.budget) policy.update(alpha * gradients['grad_theta']) # PERFORM SECOND STEP newJ_det = self.estimate_policy_performance(policy, N2, parallel=parallel, deterministic=True) J_journey += newJ_det * N2 self.budget += N2 * (newJ_det - prevJ_det ) # B += J(theta', 0) - J(theta, 0) prevJ_det = newJ_det # PERFORM THIRD STEP features, actions, rewards, J_hat, gradients = self.get_trajectories_data( policy, N3, parallel=parallel) J_journey += J_hat * N3 self.budget += N1 * (J_hat - prevJ ) # B += J(theta', sigma) - J(theta, sigma) prevJ = J_hat beta, N3, safe = self.meta_selector.select_beta( policy, gradients, self.task_prop, N3, iteration, self.budget) policy.update_w(beta * gradients['gradDeltaW']) N_old = N J_journey /= N #Check if done N_tot += N if N_tot >= self.constr.N_tot: print('Total N reached') print('End experiment') break # SAVE DATA self.save_data(filename)
def run( self, policy, use_local_stats=False, # Update task prop only with local stats parallel=True, filename=generate_filename(), verbose=False): self.use_local_stats = True self.initial_configuration = self.get_param_list(locals()) self.estimator = Estimators(self.task_prop, self.constr) N = N_old = self.constr.N_min # Total number of trajectories to take in this iteration N1, N2, N3 = self.split_trajectory_count(N) #Multiprocessing preparation if parallel: self._enable_parallel() # COMPUTE BASELINES features, actions, rewards, prevJ, gradients = self.get_trajectories_data( policy, N, parallel=parallel) #prevJ_det = self.estimate_policy_performance(policy, N, parallel=parallel, deterministic=True, get_min=False) prevJ_det = prevJ #Learning iteration = 0 N_tot = 0 start_time = time.time() J_hat = prevJ J_journey = (2 * prevJ + prevJ_det) / 3 policy_low_variance = policy while iteration < self.constr.max_iter: iteration += 1 J_det_exact = self.env.computeJ(policy.theta_mat, 0) self.make_checkpoint(locals()) # CHECKPOINT BEFORE SIGMA STEP J_journey = 0 # PRINT if verbose: if iteration % 50 == 1: print( 'IT\tN\t\tJ\t\t\tJ_DET\t\t\tTHETA\t\tSIGMA\t\t\tBUDGET' ) print(iteration, '\t', N, '\t', J_hat, '\t', prevJ_det, '\t', policy.get_theta(), '\t', policy.sigma, '\t', self.budget / N, '\t', time.time() - start_time) start_time = time.time() # PERFORM FIRST STEP features, actions, rewards, J_hat, gradients = self.get_trajectories_data( policy, N1, parallel=parallel) J_journey += J_hat * N1 if iteration > 1: self.budget += N3 * ( J_hat - prevJ) # B += J(theta, sigma') - J(theta, sigma) prevJ = J_hat alpha, N1, safe = self.meta_selector.select_alpha( policy, gradients, self.task_prop, N1, iteration, self.budget) policy.update(alpha * gradients['grad_theta']) # PERFORM THIRD STEP features, actions, rewards, J_hat, gradients = self.get_trajectories_data( policy, N3, parallel=parallel) J_journey += J_hat * N3 self.budget += N1 * (J_hat - prevJ ) # B += J(theta', sigma) - J(theta, sigma) prevJ = J_hat beta, N3, safe = self.meta_selector.select_beta( policy, gradients, self.task_prop, N3, iteration, self.budget) # policy.update_w(beta * gradients['gradDeltaW']) # COMPUTE OPTIMAL SIGMA_0 USING THE BOUND d = policy.penaltyCoeffSigma(self.task_prop.R, self.task_prop.M, self.task_prop.gamma, self.task_prop.volume) if self.budget / N2 >= -(gradients['grad_w']**2) / (4 * d): beta_minus = (1 - math.sqrt(1 - (4 * d * (-self.budget / N2)) / (gradients['grad_w']**2))) / (2 * d) beta_plus = (1 + math.sqrt(1 - (4 * d * (-self.budget / N2)) / (gradients['grad_w']**2))) / (2 * d) w_det = min(policy.w + beta_minus * gradients['grad_w'], policy.w + beta_plus * gradients['grad_w']) policy_low_variance = policies.ExpGaussPolicy( np.copy(policy.theta_mat), w_det) else: policy_low_variance = policy newJ_det = self.estimate_policy_performance(policy_low_variance, N2, parallel=parallel) J_journey += newJ_det * N2 self.budget += N2 * (newJ_det - prevJ_det ) # B += J(theta', 0) - J(theta, 0) prevJ_det = newJ_det # beta, N3, safe = self.meta_selector.select_beta(policy, gradients, self.task_prop, N3, iteration, self.budget) policy.update_w(beta * gradients['gradDeltaW']) N_old = N J_journey /= N #Check if done N_tot += N if N_tot >= self.constr.N_tot: print('Total N reached') print('End experiment') break # def signal_handler(signal, frame): # self.save_data(filename) # sys.exit(0) # # #Manual stop # signal.signal(signal.SIGINT, signal_handler) # SAVE DATA self.save_data(filename)
def run(self, policy, use_local_stats=False, parallel=True, filename=generate_filename(), verbose=False): self.use_local_stats = True self.initial_configuration = self.get_param_list(locals()) self.estimator = Estimators(self.task_prop, self.constr) N = self.constr.N_min # Total number of trajectories to take in this iteration #Multiprocessing preparation if parallel: self._enable_parallel() # COMPUTE BASELINES features, actions, rewards, prevJ, gradients = self.get_trajectories_data( policy, N, parallel=parallel) J_baseline = prevJ #Learning iteration = 0 N_tot = 0 J_journey = prevJ start_time = time.time() N1 = N // 2 N2 = N - N1 while iteration < self.constr.max_iter: iteration += 1 J_det_exact = self.evaluate(policy, deterministic=True) self.make_checkpoint(locals()) # CHECKPOINT BEFORE SIGMA STEP J_journey = 0 # PRINT if verbose: if iteration % 50 == 1: print( 'IT\tN\t\tJ\t\t\tJ_DET\t\t\tTHETA\t\tSIGMA\t\t\tBUDGET' ) print(iteration, '\t', N, '\t', prevJ, '\t', 0, '\t', policy.get_theta(), '\t', policy.sigma, '\t', self.budget / N, '\t', time.time() - start_time) start_time = time.time() # PERFORM FIRST STEP features, actions, rewards, prevJ, gradients = self.get_trajectories_data( policy, N1, parallel=parallel) J_journey += prevJ * N1 budget = prevJ - J_baseline alpha, N1, safe = self.meta_selector.select_alpha(policy, gradients, self.task_prop, N1, iteration, budget=budget) policy.update(alpha * gradients['grad_theta']) # PERFORM SECOND STEP features, actions, rewards, prevJ, gradients = self.get_trajectories_data( policy, N2, parallel=parallel) J_journey += prevJ * N2 budget = prevJ - J_baseline beta, _, _ = self.meta_selector.select_beta(policy, gradients, self.task_prop, N2, iteration, budget=budget) policy.update_w(beta * gradients['gradDeltaW']) J_journey /= N N_tot += N if N_tot >= self.constr.N_tot: print('Total N reached\nEnd experiment') break # SAVE DATA self.save_data(filename)
def biz(): csv_fname = generate_filename('ALL_BUSINESSES') + '.csv' limit = 20 offset = 0 pagination = True nxt_url = None while pagination: print('---------------------------------START: BUSINESS API GET: OFFSET {}'.format(offset)) api_url = FIVESTARS_BUSINESS_GROUP_OFFSET_API.format(limit, offset) if not nxt_url else nxt_url resp = request(api_url) if resp: jsn = resp.json() nxt_url = jsn.get('meta').get('next') if not nxt_url: pagination = False data = jsn.get('data') if data: for biz_group in data: total_biz = biz_group.get('total_businesses') bizs = biz_group.get('businesses') # print('bizs', bizs) if bizs: fstars = FiveStars(bizs) businesses = fstars.get_businesses() for biz in businesses: name = biz.get('name') addr = biz.get('address') street = addr.get('street') city = addr.get('city') state = addr.get('state') zipcode = addr.get('postal_code') phone = biz.get('phone') if phone and len(phone) == 10: phone = '(' + phone[:3] + ') ' + phone[4:] total_locs = total_biz website = biz.get('website') fb = biz.get('facebook') instagram = biz.get('instagram') tw = biz.get('twitter') yelp = biz.get('yelp') prof = biz.get('profile') desc = biz.get('description') hours = biz.get('hours') keywords = biz.get('keywords') logo = biz.get('logo') gplus = biz.get('google_plus') cat = biz.get('category') write_csv(csv_fname, [[ name, street, city, state, zipcode, phone, total_locs, website, fb, instagram, tw, yelp, prof, desc, hours, keywords, logo, gplus, cat ]], 'output') print('[+] Done >> {}'.format(name)) offset += limit print('---------------------------------END: BUSINESS API GET: OFFSET {}'.format(offset))
def scrape(division=False, csv_file=None, newcol=False): csv_fname = generate_filename('MAXPREPS') + '.csv' cookies = get_cookies() s = requests.Session() done_text = 'We do not have rankings for this selection, please try another search' if not division: pagination = True cur_page = 1 while pagination: soup = request(s, RANKINGS_URL.format(cur_page), cookies) if soup: if done_text in str(soup): break else: print( '-----------------------Page {}-----------------------' .format(cur_page)) cur_page = cur_page + 1 try: trs = soup.select('table tr')[1:] except: trs = [] for tr in trs: rp = RowParser(tr) name = rp.get_name() rating = rp.get_rating() strength = rp.get_strength() p_url = rp.get_profle_url() s_num = rp.get_search_num() prof_soup = request(s, p_url, cookies) if soup: pp = ProfileParser(s, cookies, prof_soup, s_num, name, rating, strength, p_url) data = pp.get_data() write_csv(csv_fname, data) print('[+] Done >> {}'.format(name)) else: # this is just for added state class/ranks cols if newcol: with open(csv_file, 'r') as csvfile: reader = csv.reader(csvfile) for row in reader: name = row[5] url = row[4] soup = request(s, url, cookies) if soup: fp = FootballProfile(name, soup) data = [[ row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11], row[12], row[13], row[14], row[15], row[16], row[17], row[18], fp.state_class_division_name(), fp.state_division_rank() ]] write_csv('__' + csv_file, data) print('[+] Done >> {}'.format(name)) else: for div in DIVISIONS: print( '-----------------------DIVISION: {}-----------------------' .format(div)) state_soup = request(s, RANKINGS_BY_DIVISION.format(1, div), cookies) if state_soup: state_class = get_state_class(state_soup) if state_class: for sc in state_class: clas_url = sc[1] clas_name = sc[0] print( '-----------------------STATE CLASS: {}-----------------------' .format(clas_name)) pagination = True cur_page = 1 while pagination: # '/rankings/football-fall-16/{}/division/fl/asZCeSbCLkCIkW5UXU_cOw/division-8a.htm' url = re.sub(r'/[0-9]{1,3}/', '/{}/', clas_url) soup = request(s, url.format(cur_page), cookies) if soup: if done_text in str(soup): pagination = False else: print( '-----------------------Page {}-----------------------' .format(cur_page)) cur_page = cur_page + 1 try: trs = soup.select('table tr')[1:] except: trs = [] for tr in trs: rp = RowParser(tr) name = rp.get_name() rating = rp.get_rating() strength = rp.get_strength() p_url = rp.get_profle_url() s_num = rp.get_search_num() prof_soup = request( s, p_url, cookies) if soup: pp = ProfileParser( s, cookies, prof_soup, s_num, name, rating, strength, p_url) data = pp.get_data() write_csv(csv_fname, data) print('[+] Done >> {}'.format( name)) else: pagination = True cur_page = 1 while pagination: soup = request( s, RANKINGS_BY_DIVISION.format(cur_page, div), cookies) if soup: if done_text in str(soup): pagination = False else: print( '-----------------------Page {}-----------------------' .format(cur_page)) cur_page = cur_page + 1 try: trs = soup.select('table tr')[1:] except: trs = [] for tr in trs: rp = RowParser(tr) name = rp.get_name() rating = rp.get_rating() strength = rp.get_strength() p_url = rp.get_profle_url() s_num = rp.get_search_num() prof_soup = request(s, p_url, cookies) if soup: pp = ProfileParser( s, cookies, prof_soup, s_num, name, rating, strength, p_url) data = pp.get_data() write_csv(csv_fname, data) print( '[+] Done >> {}'.format(name))