def all_with_filter(cls, contributor_id, publication_status, tags, uri, statuses): availlable_filters = { 'past': and_(cls.end_publication_date != None, cls.end_publication_date < get_current_time()), 'ongoing': and_(cls.start_publication_date <= get_current_time(), or_(cls.end_publication_date == None, cls.end_publication_date >= get_current_time())), 'coming': Disruption.start_publication_date > get_current_time() } query = cls.query.filter(and_( cls.contributor_id == contributor_id, cls.status.in_(statuses) )) if tags: query = query.filter(cls.tags.any(Tag.id.in_(tags))) if uri: query = query.join(cls.impacts) query = query.filter(Impact.status == 'published') query = query.join(Impact.objects) #Here add a new query to find impacts with line =_section having uri as line, start_point or end_point filters = [] alias_line = aliased(PTobject) alias_start_point = aliased(PTobject) alias_end_point = aliased(PTobject) alias_route = aliased(PTobject) alias_via = aliased(PTobject) query_line_section = query query_line_section = query_line_section.join(PTobject.line_section) query_line_section = query_line_section.join(alias_line, LineSection.line_object_id == alias_line.id) filters.append(alias_line.uri == uri) query_line_section = query_line_section.join(PTobject, LineSection.object_id == PTobject.id) query_line_section = query_line_section.join(alias_start_point, LineSection.start_object_id == alias_start_point.id) filters.append(alias_start_point.uri == uri) query_line_section = query_line_section.join(alias_end_point, LineSection.end_object_id == alias_end_point.id) filters.append(alias_end_point.uri == uri) query_line_section = query_line_section.join(alias_route, LineSection.routes) filters.append(alias_route.uri == uri) query_line_section = query_line_section.join(alias_via, LineSection.via) filters.append(alias_via.uri == uri) query_line_section = query_line_section.filter(or_(*filters)) query = query.filter(PTobject.uri == uri) publication_status = set(publication_status) if len(publication_status) == len(publication_status_values): #For a query by uri use union with the query for line_section if uri: query = query.union_all(query_line_section) else: filters = [availlable_filters[status] for status in publication_status] query = query.filter(or_(*filters)) #For a query by uri use union with the query for line_section if uri: query_line_section = query_line_section.filter(or_(*filters)) query = query.union_all(query_line_section) return query.order_by(cls.end_publication_date)
def all_with_filter(cls, contributor_id, publication_status, ends_after_date, ends_before_date, tags, uri, line_section, statuses): availlable_filters = { 'past': and_(cls.end_publication_date != None, cls.end_publication_date < get_current_time()), 'ongoing': and_(cls.start_publication_date <= get_current_time(), or_(cls.end_publication_date == None, cls.end_publication_date >= get_current_time())), 'coming': Disruption.start_publication_date > get_current_time() } query = cls.query.filter(and_( cls.contributor_id == contributor_id, cls.status.in_(statuses) )) if ends_after_date: query = query.filter(cls.end_publication_date >= ends_after_date) if ends_before_date: query = query.filter(cls.end_publication_date <= ends_before_date) if tags: query = query.filter(cls.tags.any(Tag.id.in_(tags))) if uri: query = query.join(cls.impacts) query = query.filter(Impact.status == 'published') query = query.join(Impact.objects) #Here add a new query to find impacts with line_section having uri as line if line_section : query_line_section = query query_line_section = query_line_section.filter(and_(PTobject.type == "line_section", PTobject.uri.like(uri + ":%"))) query = query.filter(PTobject.uri == uri) publication_status = set(publication_status) if len(publication_status) == len(publication_status_values): #For a query by uri use union with the query for line_section if uri and line_section: query = query.union(query_line_section) else: filters = [availlable_filters[status] for status in publication_status] query = query.filter(or_(*filters)) #For a query by uri use union with the query for line_section if uri and line_section: query_line_section = query_line_section.filter(or_(*filters)) query = query.union(query_line_section) return query.order_by(cls.end_publication_date, cls.id)
def populate_db(): with open('data/states.csv', mode='r') as infile: reader = csv.reader(infile) mydict = {rows[0]: {'pledged':rows[1], 'unpledged':rows[2], 'date': rows[4]} for rows in reader} for state, delegates in mydict.items(): s = models.State(state=state) s.pledged_available = int(delegates['pledged']) s.unpledged_available = int(delegates['unpledged']) try: s.election_date = datetime.strptime(delegates['date'], '%m/%d/%Y') except ValueError as e: print 439750983246709479074092760974092764097460927903467 print delegates['date'] s.total_available = s.pledged_available + s.unpledged_available s.clinton_percentage = 0 s.sanders_percentage = 0 s.clinton_delegates = 0 s.sanders_delegates = 0 s.last_updated = get_current_time() db.session.add(s) db.session.commit() clinton = models.Candidate(last_name='Clinton', first_name='Hillary') sanders = models.Candidate(last_name='Sanders', first_name='Bernie') db.session.add(clinton) db.session.add(sanders) db.session.commit() get_charts() get_results()
def get_results(): with open('data/results.csv', mode='r') as infile: reader = csv.reader(infile) mydict = {rows[0]: {'clinton_pledged':rows[1], 'clinton_unpledged':rows[2], 'sanders_pledged':rows[3], 'sanders_unpledged':rows[4], 'clinton_percentage':rows[5], 'sanders_percentage':rows[6]} for rows in reader} clinton = models.Candidate.query.filter_by(last_name='Clinton').first() sanders = models.Candidate.query.filter_by(last_name='Sanders').first() for state, results in mydict.items(): if models.State.query.filter_by(state=state).first(): s = models.State.query.filter_by(state=state).first() s.last_updated = get_current_time() s.clinton_pledged_delegates_results = int(results['clinton_pledged']) s.sanders_pledged_delegates_results = int(results['sanders_pledged']) s.clinton_unpledged_delegates_results = int(results['clinton_unpledged']) s.sanders_unpledged_delegates_results = int(results['sanders_unpledged']) db.session.add(s) s.clinton_percentage_results = float(results['clinton_percentage']) s.sanders_percentage_results = float(results['sanders_percentage']) clinton.pledged_delegates += int(results['clinton_pledged']) clinton.unpledged_delegates += int(results['clinton_unpledged']) sanders.pledged_delegates += int(results['sanders_pledged']) sanders.unpledged_delegates += int(results['sanders_unpledged']) clinton.total_delegates = clinton.pledged_delegates + clinton.unpledged_delegates sanders.total_delegates = sanders.pledged_delegates + sanders.unpledged_delegates db.session.commit()
def traffic_report_filter(cls, contributor_id): query = cls.query.filter(cls.status == "published") query = query.join(Disruption) query = query.filter(Disruption.contributor_id == contributor_id) query = query.filter( between(get_current_time(), Disruption.start_publication_date, Disruption.end_publication_date) ) return query.all()
def collection_qa_analyzer(col_name, env='active', fields=None): # parse configuration file cfg_fpath = os.path.join(basepath, "db_and_fpath.cfg") parser = ConfigParser() parser.read(cfg_fpath) # get detailed info from configuration mongo_uri = parser.get(env, 'uri') db_name = parser.get(env, 'db_name') io_fpath = parser.get(env, 'io_fpath') # get current time in string current_time = get_current_time(flag='short') # create directories if not existing, and return the collection's path col_fpath = make_dirs(io_fpath, current_time, db_name, col_name) # get all file paths that are required basic_stat_fpath = os.path.join(col_fpath, 'basic_stat.json') domain_fpath = os.path.join(col_fpath, 'domain.json') existing_checking_fpath = os.path.join(col_fpath, 'existing.json') field_fpath = os.path.join(col_fpath, 'fields.json') type_checking_fpath = os.path.join(col_fpath, 'type_checking.json') nullable_checking_fpath = os.path.join(col_fpath, 'nullable.json') empty_checking_fpath = os.path.join(col_fpath, 'empty.json') coverage_fpath = os.path.join(col_fpath, 'coverage.json') merger_fpath = os.path.join(col_fpath, 'merger.txt') # MongoEye mongoEye = MongoEye(mongo_uri, db_name, col_name) # get detailed STAT mongoEye.save_existing_checking(existing_checking_fpath) mongoEye.save_fields(existing_checking_fpath, field_fpath) mongoEye.save_type_checking(type_checking_fpath, field_fpath) mongoEye.save_nullable_checking(field_fpath, nullable_checking_fpath) mongoEye.save_empty_checking(type_checking_fpath, empty_checking_fpath) mongoEye.save_coverage(existing_checking_fpath, nullable_checking_fpath, empty_checking_fpath, coverage_fpath) mongoEye.merger(field_fpath, type_checking_fpath, existing_checking_fpath, nullable_checking_fpath, empty_checking_fpath, coverage_fpath, merger_fpath) # get basic STAT mongoEye.save_domain_counters(domain_fpath) mongoEye.get_basic_stat(merger_fpath, basic_stat_fpath, domain_fpath) # test count unique domains #print mongoEye.count_domains() #=============================================================================== # 获得某一个字段值的分布(distribution) #=============================================================================== if fields: field_dist_collector(mongoEye, fields, col_fpath)
def output(self, key, obj): current_datetime = get_current_time() is_future = False for application_period in obj.application_periods: if current_datetime >= application_period.start_date and current_datetime <= application_period.end_date: return 'active' if current_datetime <= application_period.start_date: is_future = True if is_future: return 'future' return 'past'
def my_predictions(): is_logged_in, user = is_user_logged_in(session) if not is_logged_in: return redirect(url_for("index")) user_info = user.to_json() return render_template( "my-predictions.html", user=user_info, fixtures=football_api_client.get_all_fixtures(), points=get_points_for_user(user_info["predictions"]), current_time=get_current_time() )
def profile(): form = ProfileForm(obj=current_user) if form.validate_on_submit(): form.populate_obj(user) user.update_at = get_current_time() db.session.commit() flash('Public profile updated.', 'success') return render_template('user/profile.html', form=form)
def save_db(temp): timestamp = int(time.time()) datetime = get_current_time() pushed = 0 query = 'insert into measure values (null, %s, "%s", "%s", %s, %s)' % (timestamp, datetime, ROOM, pushed, temp) try: conn = sqlite3.connect(get_abs_path() + '/pi-temp.db') conn.execute(query) conn.commit() conn.close() except: logging.error('Error when saving measure in database') pass
def get_context(self): # get the timezone from the GET params (default to UTC) tz = self.request.get('tz', default_value='UTC') # attempt to parse timezone try: timezone = get_timezone(tz) # return error if invalid timezone except pytz.UnknownTimeZoneError: # set HTTP status and return error msg self.response.set_status(400) return {'error': 'Unknown timezone "%s"' % tz} # add timezone name to response response = {'tz_name': timezone.zone} # add utc offset to response response['tz_offset'] = get_utcoffset_as_string(timezone) # add current time to response response['datetime'] = get_current_time(tz) return response
def insert_or_update_post(self, post): if post is None: return author = get_post_author_id(post['author']) board = post['board'] post_time = get_post_time(post['date']) title = post['article_title'] ip = post['ip'] messages = post['messages'] post_id = post['article_id'] content = post['content'] url = post['url'] self.add_user(author) self.add_board(board) with self.__post_insertion_lock: self.__execute_write( ''' INSERT OR REPLACE INTO `posts` ( `id`, `board`, `post_id`, `author`, `date_time`, `title`, `web_url`, `ip` ) VALUES ( ( SELECT `id` FROM `posts` WHERE `board` = ( SELECT `id` FROM `boards` WHERE `name` = :board ) AND `post_id` = :post_id ), ( SELECT `id` FROM `boards` WHERE `name` = :board ), :post_id, ( SELECT `id` FROM `users` WHERE `username` = :author ), :date_time, :title, :web_url, :ip ); ''', { 'board': board, 'post_id': post_id, 'author': author, 'date_time': post_time, 'title': title, 'web_url': url, 'ip': ip }) self.__execute_write( ''' INSERT OR REPLACE INTO `posts_content` (`id`, `post`, `content`) VALUES ( ( SELECT `id` FROM `posts_content` WHERE `post` = ( SELECT `id` FROM `posts` WHERE `board` = ( SELECT `id` FROM `boards` WHERE `name` = :board ) AND `post_id` = :post_id ) ), ( SELECT `id` FROM `posts` WHERE `board` = ( SELECT `id` FROM `boards` WHERE `name` = :board ) AND `post_id` = :post_id ), :content ); ''', { 'board': board, 'post_id': post_id, 'content': content }) # delete existing pushes self.__execute_write( ''' DELETE FROM `pushes` WHERE `post` = ( SELECT `id` FROM `posts` WHERE `board` = ( SELECT `id` FROM `boards` WHERE `name` = :board ) AND `post_id` = :post_id ); ''', { 'board': board, 'post_id': post_id }) year = get_post_year(post['date']) if year == 0: return for push in messages: author = push['push_userid'] push_content = push['push_content'] push_tag = push['push_tag'] ip, push_time = get_push_ip_time(year, push['push_ipdatetime']) if push_time < post_time: _, push_time = get_push_ip_time(year + 1, push['push_ipdatetime']) self.add_user(author) self.__execute_write( ''' INSERT INTO `pushes` ( `post`, `type`, `author`, `content`, `ip`, `date_time` ) VALUES ( ( SELECT `id` FROM `posts` WHERE `board` = ( SELECT `id` FROM `boards` WHERE `name` = :board ) AND `post_id` = :post_id ), :type , ( SELECT `id` FROM `users` WHERE `username` = :author ), :content , :ip , :date_time ) ''', { 'board': board, 'post_id': post_id, 'type': push_tag, 'author': author, 'content': push_content, 'ip': ip, 'date_time': push_time }) self.__execute_write( ''' INSERT OR REPLACE INTO `crawled_posts` (`id`, `post`, `date_time`) VALUES ( ( SELECT `id` FROM `crawled_posts` WHERE `post` = ( SELECT `id` FROM `posts` WHERE `board` = ( SELECT `id` FROM `boards` WHERE `name` = :board ) AND `post_id` = :post_id ) ), ( SELECT `id` FROM `posts` WHERE `board` = ( SELECT `id` FROM `boards` WHERE `name` = :board ) AND `post_id` = :post_id ), :date_time ); ''', { 'board': board, 'post_id': post_id, 'date_time': get_current_time() })
def get_time_left(self, current_datetime=utils.get_current_time()): return (self.expiry - current_datetime).total_seconds() / (day * 365)
def listen_print_loop(self, responses): """Iterates through server responses and prints them. The responses passed is a generator that will block until a response is provided by the server. Each response may contain multiple results, and each result may contain multiple alternatives; for details, see https://goo.gl/tjCPAU. Here we print only the transcription for the top alternative of the top result. In this case, responses are provided for interim results as well. If the response is an interim one, print a line feed at the end of it, to allow the next result to overwrite it, until the response is a final one. For the final one, print a newline to preserve the finalized transcription. """ info("Entered listen_print_loop") for response in responses: if (get_current_time() - self.stream.start_time > self.streaming_limit): self.stream.start_time = get_current_time() break if not response.results: continue # The `results` list is consecutive. For streaming, we only care about # the first result being considered, since once it's `is_final`, it # moves on to considering the next utterance. result = response.results[0] if not result.alternatives: continue # Display the transcription of the top alternative. transcript = result.alternatives[0].transcript result_seconds = 0 result_micros = 0 if result.result_end_time.seconds: result_seconds = result.result_end_time.seconds if result.result_end_time.microseconds: result_micros = result.result_end_time.microseconds self.stream.result_end_time = int((result_seconds * 1000) + (result_micros / 1000)) corrected_time = ( self.stream.result_end_time - self.stream.bridging_offset + (self.streaming_limit * self.stream.restart_counter)) if self.websocket is not None: message = json.dumps({ "transcript": transcript, "is_final": result.is_final }) asyncio.run(self.publish_to_websocket(message)) self.publish_transcript(transcript, result.is_final, "Google") if result.is_final: self.stream.is_final_end_time = self.stream.result_end_time self.stream.last_transcript_was_final = result.is_final
parser.add_argument('--dataset', default='coco', choices=['tejani', 'coco'], type=str, help="Dataset name") parser.add_argument('--checkpoint', default='0.0', type=str, help="Checkpoint name in format: `epoch.iteration`") parser.add_argument('--gpu', default='0', type=str, help="GPU id") return parser.parse_args() args = parse_arg() cfg = config.network[args.dataset]['cfg'] log_dir = opj(config.LOG_ROOT, get_current_time()) writer = SummaryWriter(log_dir=log_dir) os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu def train(epoch, trainloader, yolo, optimizer): """Training wrapper @Args epoch: (int) training epoch trainloader: (Dataloader) train data loader yolo: (nn.Module) YOLOv3 model optimizer: (optim) optimizer """ yolo.train() tbar = tqdm(trainloader, ncols=80)
def collect_logs_per_pod(self, podstatus): containers = utils.get_container_names_per_pod(podstatus.namespace, podstatus.name) for container in containers: log_file_name = "{}-{}-{}".format(podstatus.name, container, utils.get_current_time()) command = "kubectl logs {} --container={} --timestamps=true " \ "--since={} --limit-bytes={} -n {} > {}/{}.log" cmd = command.format( podstatus.name, container, self.since, self.size_limit, podstatus.namespace, self.sysdump_dir_name, log_file_name) try: subprocess.check_output( cmd, shell=True, stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as exc: if exc.returncode != 0: log.error("Error: {}. Could not collect log file: {}" .format(exc, log_file_name)) else: log.info("collected log file: {}".format(log_file_name)) # Only get logs from previous containers if pod has restarted. # Need to get the pod to access its restartCount. podCommandPreFormatted = "kubectl get pod {} -n {} -o json" podCmd = podCommandPreFormatted.format( podstatus.name, podstatus.namespace, ) try: podOutput = subprocess.check_output( podCmd, shell=True, stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as exc: if exc.returncode != 0: log.debug("Debug {}: could not get pod {}").format( exc, podstatus[0]) else: # Examine JSON output to see if restartCount > 0 decodedPodOutput = podOutput.decode() jsonOutput = json.loads(decodedPodOutput) containerStatuses = jsonOutput['status']['containerStatuses'] gatherPrevLogs = False for value in containerStatuses: restartCount = value['restartCount'] if int(restartCount) > 0: gatherPrevLogs = True break if gatherPrevLogs: log_file_name_previous = "{0}-previous".format( log_file_name) command = "kubectl logs --previous --timestamps=true " \ "--container={} " \ "--since={} " \ "--limit-bytes={} -n {} {} > {}/{}.log" cmd = command.format(container, self.since, self.size_limit, podstatus.namespace, podstatus.name, self.sysdump_dir_name, log_file_name_previous) try: subprocess.check_output( cmd, shell=True, stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as exc: if exc.returncode != 0: log.debug( "Debug: {}. Could not collect previous " "log for '{}': {}" .format(exc, podstatus.name, log_file_name)) else: log.info("collected log file: {}".format( log_file_name_previous)) else: log.debug("no previous pod logs to gather for " "pod/container {}/{}".format( podstatus.name, container))
if args.epochs: SETTINGS['EPOCHS'] = args.epochs if args.mini == "True": SETTINGS['NUM_CLASSES'] = 11 SETTINGS['DATA_PATHS']['TRAIN_CSV'] = "mini_train.csv" SETTINGS['DATA_PATHS']['TEST_CSV'] = "mini_test.csv" if args.lr: SETTINGS["LR"] = args.lr if args.decay: SETTINGS["DECAY"] = args.decay SETTINGS["WLOSS"] = args.wloss == "True" SETTINGS["TRANSFORMER"] = args.transformer if args.batch: SETTINGS["BATCH_SIZE"] = args.batch TIME = get_current_time() # Load and transform data transform = get_transformers()[SETTINGS["TRANSFORMER"]] dataset = Loader(SETTINGS['DATA_PATHS']['TRAIN_CSV'], SETTINGS['DATA_PATHS']['DATASET_PATH'], transform=transform) test_dataset = Loader(SETTINGS['DATA_PATHS']['TEST_CSV'], SETTINGS['DATA_PATHS']['DATASET_PATH'], transform=transform) # Train k models and keep the best logging.info("Settings: {}".format(str(SETTINGS))) best_model = train(dataset) plot_loss(best_model, SETTINGS) test(best_model, test_dataset)
def train(config): # load data user_pos_train, user_pos_test = load_data(config) all_users = list(user_pos_train.keys()) all_users.sort() user_num = config['user_num'] item_num = config['item_num'] if not os.path.exists(config['output_dir']): os.mkdir(config['output_dir']) with open(os.path.join(config['output_dir'], 'config.json'), 'w') as fout: print(json.dumps(config), file=fout) train_log = open(os.path.join(config['output_dir'], 'train_log.txt'), 'w') # build model generator = GEN(config) discriminator = DIS(config) saver = tf.compat.v1.train.Saver(max_to_keep=1) model_path = os.path.join(config['output_dir'], 'model/SD-GAR') os.environ["CUDA_VISIBLE_DEVICES"] = config['CUDA_VISIBLE_DEVICES'] config_tf = tf.compat.v1.ConfigProto() config_tf.gpu_options.allow_growth = True sess = tf.compat.v1.Session(config=config_tf) # init variable sess.run(tf.compat.v1.global_variables_initializer()) # init embeddings print("%s; initializing embeddings and constructing alias table..." % get_current_time()) prob_users, prob_items = get_init_embeddings(config) alias_table = AliasTable(prob_users, prob_items) sampler_d = SamplerD(config, alias_table=alias_table, generator=generator, sess=sess) print("%s; finished" % get_current_time()) # minimax training best, best_gen = 0., 0. global Train_d_cnt, Gen_data_for_d_time, Train_d_time global Train_g_cnt, Gen_data_for_g_time, Train_g_time mretic_name = [ 'P@3', 'P@5', 'P@10', 'P@50', 'NDCG@3', 'NDCG@5', 'NDCG@10', 'NDCG@50', 'MRR' ] for epoch in range(76): # train discriminator if epoch > 0: Train_d_cnt += 1 batch_num = 0 # the loss incorporates four parts including total loss, positive sample loss, negtive sample loss and regularization loss_arr = np.array([0.] * 4) time_start = time.time() sampler_d.generate_data(user_pos_train, config['dis_sample_num'], shuffle=True) sampler_d.generate_neg_scores() Gen_data_for_d_time += time.time() - time_start data_len = len(sampler_d.data) time_start = time.time() index = 0 while index < data_len: batch = sampler_d.get_next_batch(config['batch_size']) users = batch['users'] pos_items = batch['pos_items'] neg_items = batch['neg_items'] neg_scores = batch['neg_scores'] index += config['batch_size'] _, batch_loss_list = sess.run( [discriminator.get_update(), discriminator.get_loss()], feed_dict={ discriminator.u: users, discriminator.pos_i: pos_items, discriminator.neg_i: neg_items, discriminator.g_scores: neg_scores }) batch_loss_list[1] = np.mean(batch_loss_list[1]) batch_loss_list[2] = np.mean(batch_loss_list[2]) batch_loss_arr = np.array(batch_loss_list) loss_arr += batch_loss_arr batch_num += 1 Train_d_time += time.time() - time_start loss_arr = loss_arr / batch_num curr_time = get_current_time() buf = "%s; epoch: %s; loss: %s; pos_loss: %s; neg_loss: %s; regular_loss: %s" % ( curr_time, epoch, loss_arr[0], loss_arr[1], loss_arr[2], loss_arr[3]) output_to_file(buf, train_log) if epoch % 5 == 0: result = eval(sess, discriminator, user_pos_train, user_pos_test) curr_time = get_current_time() buf = "\t%s; metrics: \t%s" % (curr_time, '\t'.join( ["%7s" % x for x in mretic_name])) output_to_file(buf, train_log) buf = "\t%s; performance:\t%s" % (curr_time, '\t'.join( ["%.5f" % x for x in result])) output_to_file(buf, train_log) ndcg_50 = result[7] if ndcg_50 > best: buf = '\tbest ndcg@50, saving the current model' output_to_file(buf, train_log) best = ndcg_50 saver.save(sess, model_path) f_gen_embeddings = open( os.path.join(config['output_dir'], 'gen_embeddings.txt'), 'wb') pickle.dump( [alias_table.prob_users, alias_table.prob_items], f_gen_embeddings) if epoch % 5 == 0: Train_g_cnt += 1 print("%s; computing partition function..." % get_current_time()) z_u, d_logits = get_partition_funciton(sess, discriminator, generator, alias_table, config) # update user embeddings print("%s; computing u..." % get_current_time()) prob_users = get_new_user_embeddings(sess, discriminator, generator, z_u, alias_table, config) print("%s; update alias table u..." % get_current_time()) time_start = time.time() # update user alias table alias_table.update_users(prob_users) Train_g_time += time.time() - time_start print("%s; finish updating..." % get_current_time()) print("%s; computing v..." % get_current_time()) # update item embeddings prob_items = get_new_item_embeddings(sess, discriminator, generator, z_u, alias_table, config) print("%s; update alias table v..." % get_current_time()) # update item alias table time_start = time.time() alias_table.update_items(prob_items) Train_g_time += time.time() - time_start print("%s; finish updating..." % get_current_time()) output_to_file( "cost on generating data for d: %s" % (Gen_data_for_d_time / Train_d_cnt), train_log) output_to_file("cost on training d: %s" % (Train_d_time / Train_d_cnt), train_log) output_to_file( "cost on generating data for g: %s" % (Gen_data_for_g_time / Train_g_cnt), train_log) output_to_file("cost on training g: %s" % (Train_g_time / Train_g_cnt), train_log) train_log.close()
def update_active(self, active_id, phone, title, profile, cost, address, details_address, time, url, active_type): self.__conn__.execute("DELETE FROM %s where %s like '%s' " % (active_table, Active_column.ID, active_id)) self.__conn__.commit() self.__conn__.execute('INSERT INTO %s (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)' \ % (active_table, Active_column.ID, Active_column.PHONE, Active_column.TITLE, Active_column.PROFILE, Active_column.COST, Active_column.ADDRESS, Active_column.DETAILS_ADDRESS, Active_column.TIME , Active_column.URL, Active_column.TYPE, Active_column.POST_TIME, Active_column.STATUS), (active_id, phone, title, profile, cost, address, details_address, time, url, active_type, str(utils.get_current_time()), "1")) self.__conn__.commit()
send_mail('*****@*****.**', book_image=img_binary) # 4. Sample next paper next_paper_id = sample_next_paper(paper_id) print('Next paper_id {}'.format(next_paper_id)) # 5. Store the next paper-to-read in database doc = {'paper_id': next_paper_id, 'date': get_current_time()} mongo_obj.push(USER_EMAIL, doc) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('-r', '--reset', action='store_true', default=False, help="If True, inserts the seed paper data in the db") args = vars(parser.parse_args()) if args['reset']: # Reset the data in mongodb mongo_obj.push('*****@*****.**', doc={ 'paper_id': '1810.04805', 'date': get_current_time() }) walk()
parser.add_argument('--channels', default=3, type=int) parser.add_argument('--dataset', default='kth', help='dataset to train with') parser.add_argument('--max_step', type=int, default=20, help='maximum distance between frames') parser.add_argument('--sd_weight', type=float, default=0.0001, help='weight on adversarial loss') parser.add_argument('--sd_nf', type=int, default=100, help='number of layers') parser.add_argument('--content_model', default='dcgan_unet', help='model type (dcgan | dcgan_unet | vgg_unet)') parser.add_argument('--pose_model', default='dcgan', help='model type (dcgan | unet | resnet)') parser.add_argument('--data_threads', type=int, default=24, help='number of parallel data loading threads') parser.add_argument('--normalize', action='store_true', help='if true, normalize pose vector') parser.add_argument('--data_type', default='drnet', help='speed up data loading for drnet training') opt = parser.parse_args() opt.dataset = 'kth' rand_str = '@'+str(uuid.uuid4())[:4] name = utils.get_current_time() + rand_str + os.sep + 'content_model=%s-pose_model=%s-content_dim=%d-pose_dim=%d-max_step=%d-sd_weight=%.3f-lr=%.3f-sd_nf=%d-normalize=%s' % (opt.content_model, opt.pose_model, opt.content_dim, opt.pose_dim, opt.max_step, opt.sd_weight, opt.lr, opt.sd_nf, opt.normalize) opt.log_dir = '%s/%s%dx%d/%s' % (opt.log_dir, opt.dataset, opt.image_width, opt.image_width, name) if not os.path.exists(opt.log_dir): os.makedirs('%s/rec/' % opt.log_dir, exist_ok=True) copyfile('./train_drnet.py', opt.log_dir+os.sep+'train_drnet.py') copyfile('./train_lstm.py', opt.log_dir+os.sep+'train_lstm.py') # checkpoint = torch.load('%s/model.pth' % opt.log_dir, map_location='cpu') # print('checkpoint: ', checkpoint) has_cuda = torch.cuda.is_available() device = torch.device("cuda" if has_cuda else "cpu") os.makedirs('%s/rec/' % opt.log_dir, exist_ok=True) os.makedirs('%s/analogy/' % opt.log_dir, exist_ok=True)
def post(self): """ Retrieve corresponding value from the test report. --- tags: - OCR parameters: - in: body name: image type: string required: true example: R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 - in: body name: search_terms type: array items: type: string required: true example: ["fasting","blood","sugar"] responses: 200: description: A corresponding value has been extracted successfully properties: extracted_value: type: string description: Retrieved value from the report """ req = request.get_json(force=True) if not req.get("image") or not req.get("search_terms"): return "Invalid image or search terms" session_filename = utils.get_current_time() reader_module_path = os.path.dirname(os.path.realpath(__file__)) origin_img_base64 = req["image"] origin_img_fp = os.path.join(reader_module_path, 'img-origin/' + session_filename + '.jpg') origin_img = Image.open(BytesIO(base64.b64decode(origin_img_base64))) origin_img.save(origin_img_fp) hocr_filepath = os.path.join(reader_module_path, 'hocr-files/' + session_filename) pytesseract.run_tesseract( origin_img_fp, hocr_filepath, extension="box", lang=None, config="hocr --psm 7 tessedit_char_whitelist=0123456789") search_terms = tuple(req["search_terms"]) # print(search_terms) try: hocr_result = hocr_search.parse_hocr(search_terms, hocr_filepath + '.hocr') img_width, img_height = origin_img.size cropped_image = origin_img.crop( utils.calc_result_box(hocr_result, img_width)) cropped_img_fp = os.path.join(reader_module_path, 'cropped-imgs/') cropped_image.save(cropped_img_fp + session_filename + ".jpg", "jpeg") response = google_vision.get_value(cropped_img_fp + session_filename + ".jpg") res_detail = {"extracted_value": response} return (res_detail) except Exception as e: error_detail = {"error": e} abort(500, message="Search terms did not match the document")
def main(args): #################### # Arguments gpu = args.gpu model_name = args.model initial_tree_sampling = args.initial_tree_sampling path_config = args.config data_augmentation = args.data_augmentation trial_name = args.name actiontype = args.actiontype max_epoch = args.max_epoch dev_size = args.dev_size # Check assert actiontype in ["train", "evaluate"] if actiontype == "train": assert max_epoch > 0 assert len(initial_tree_sampling.split("_")) == 3 for type_ in initial_tree_sampling.split("_"): assert type_ in ["X", "BU", "TD", "RB", "LB", "RB2"] assert initial_tree_sampling.split("_")[2] != "X" assert initial_tree_sampling.split("_")[1] != "RB2" assert initial_tree_sampling.split("_")[2] != "RB2" if trial_name is None or trial_name == "None": trial_name = utils.get_current_time() #################### # Path setting config = utils.Config(path_config) basename = "%s.%s.%s.aug_%s.%s" \ % (model_name, initial_tree_sampling, utils.get_basename_without_ext(path_config), data_augmentation, trial_name) if actiontype == "train": path_log = os.path.join(config.getpath("results"), basename + ".training.log") elif actiontype == "evaluate": path_log = os.path.join(config.getpath("results"), basename + ".evaluation.log") path_train = os.path.join(config.getpath("results"), basename + ".training.jsonl") path_valid = os.path.join(config.getpath("results"), basename + ".validation.jsonl") path_snapshot = os.path.join(config.getpath("results"), basename + ".model") path_pred = os.path.join(config.getpath("results"), basename + ".evaluation.ctrees") path_eval = os.path.join(config.getpath("results"), basename + ".evaluation.json") utils.set_logger(path_log) #################### # Random seed random_seed = trial_name random_seed = utils.hash_string(random_seed) random.seed(random_seed) np.random.seed(random_seed) cuda.cupy.random.seed(random_seed) #################### # Log so far utils.writelog("gpu=%d" % gpu) utils.writelog("model_name=%s" % model_name) utils.writelog("initial_tree_sampling=%s" % initial_tree_sampling) utils.writelog("path_config=%s" % path_config) utils.writelog("data_augmentation=%s" % data_augmentation) utils.writelog("trial_name=%s" % trial_name) utils.writelog("actiontype=%s" % actiontype) utils.writelog("max_epoch=%s" % max_epoch) utils.writelog("dev_size=%s" % dev_size) utils.writelog("path_log=%s" % path_log) utils.writelog("path_train=%s" % path_train) utils.writelog("path_valid=%s" % path_valid) utils.writelog("path_snapshot=%s" % path_snapshot) utils.writelog("path_pred=%s" % path_pred) utils.writelog("path_eval=%s" % path_eval) utils.writelog("random_seed=%d" % random_seed) #################### # Data preparation begin_time = time.time() train_databatch = dataloader.read_rstdt("train", relation_level="coarse-grained", with_root=False) test_databatch = dataloader.read_rstdt("test", relation_level="coarse-grained", with_root=False) vocab_word = utils.read_vocab( os.path.join(config.getpath("data"), "rstdt-vocab", "words.vocab.txt")) vocab_postag = utils.read_vocab( os.path.join(config.getpath("data"), "rstdt-vocab", "postags.vocab.txt")) vocab_deprel = utils.read_vocab( os.path.join(config.getpath("data"), "rstdt-vocab", "deprels.vocab.txt")) if data_augmentation: external_train_databatch = dataloader.read_ptbwsj_wo_rstdt( with_root=False) # Remove documents with only one leaf node filtering_function = lambda d, i: len(d.batch_edu_ids[i]) == 1 external_train_databatch = utils.filter_databatch( external_train_databatch, filtering_function) end_time = time.time() utils.writelog("Loaded the corpus. %f [sec.]" % (end_time - begin_time)) #################### # Hyper parameters word_dim = config.getint("word_dim") postag_dim = config.getint("postag_dim") deprel_dim = config.getint("deprel_dim") lstm_dim = config.getint("lstm_dim") mlp_dim = config.getint("mlp_dim") n_init_epochs = config.getint("n_init_epochs") negative_size = config.getint("negative_size") batch_size = config.getint("batch_size") weight_decay = config.getfloat("weight_decay") gradient_clipping = config.getfloat("gradient_clipping") optimizer_name = config.getstr("optimizer_name") utils.writelog("word_dim=%d" % word_dim) utils.writelog("postag_dim=%d" % postag_dim) utils.writelog("deprel_dim=%d" % deprel_dim) utils.writelog("lstm_dim=%d" % lstm_dim) utils.writelog("mlp_dim=%d" % mlp_dim) utils.writelog("n_init_epochs=%d" % n_init_epochs) utils.writelog("negative_size=%d" % negative_size) utils.writelog("batch_size=%d" % batch_size) utils.writelog("weight_decay=%f" % weight_decay) utils.writelog("gradient_clipping=%f" % gradient_clipping) utils.writelog("optimizer_name=%s" % optimizer_name) #################### # Model preparation cuda.get_device(gpu).use() # Initialize a model utils.mkdir(os.path.join(config.getpath("data"), "caches")) path_embed = config.getpath("pretrained_word_embeddings") path_caches = os.path.join( config.getpath("data"), "caches", "cached." + os.path.basename(path_embed) + ".npy") if os.path.exists(path_caches): utils.writelog("Loading cached word embeddings ...") initialW = np.load(path_caches) else: initialW = utils.read_word_embedding_matrix(path=path_embed, dim=word_dim, vocab=vocab_word, scale=0.0) np.save(path_caches, initialW) if model_name == "spanbasedmodel": # Span-based model w/ template features template_feature_extractor = models.TemplateFeatureExtractor( databatch=train_databatch) utils.writelog("Template feature size=%d" % template_feature_extractor.feature_size) if actiontype == "train": for template in template_feature_extractor.templates: dim = template_feature_extractor.template2dim[template] utils.writelog("Template feature #%s %s" % (dim, template)) model = models.SpanBasedModel( vocab_word=vocab_word, vocab_postag=vocab_postag, vocab_deprel=vocab_deprel, word_dim=word_dim, postag_dim=postag_dim, deprel_dim=deprel_dim, lstm_dim=lstm_dim, mlp_dim=mlp_dim, initialW=initialW, template_feature_extractor=template_feature_extractor) elif model_name == "spanbasedmodel2": # Span-based model w/o template features model = models.SpanBasedModel2(vocab_word=vocab_word, vocab_postag=vocab_postag, vocab_deprel=vocab_deprel, word_dim=word_dim, postag_dim=postag_dim, deprel_dim=deprel_dim, lstm_dim=lstm_dim, mlp_dim=mlp_dim, initialW=initialW) else: raise ValueError("Invalid model_name=%s" % model_name) utils.writelog("Initialized the model ``%s''" % model_name) # Load pre-trained parameters if actiontype != "train": serializers.load_npz(path_snapshot, model) utils.writelog("Loaded trained parameters from %s" % path_snapshot) model.to_gpu(gpu) #################### # Decoder preparation decoder = decoders.IncrementalCKYDecoder() #################### # Initializer preparation sampler = treesamplers.TreeSampler(initial_tree_sampling.split("_")) #################### # Training / evaluation if actiontype == "train": with chainer.using_config("train", True): if dev_size > 0: # Training with cross validation train_databatch, dev_databatch = dataloader.randomsplit( n_dev=dev_size, databatch=train_databatch) with open( os.path.join(config.getpath("results"), basename + ".valid_gold.ctrees"), "w") as f: for sexp in dev_databatch.batch_nary_sexp: f.write("%s\n" % " ".join(sexp)) else: # Training with the full training set dev_databatch = None if data_augmentation: train_databatch = utils.concat_databatch( train_databatch, external_train_databatch) training.train( model=model, decoder=decoder, sampler=sampler, max_epoch=max_epoch, n_init_epochs=n_init_epochs, negative_size=negative_size, batch_size=batch_size, weight_decay=weight_decay, gradient_clipping=gradient_clipping, optimizer_name=optimizer_name, train_databatch=train_databatch, dev_databatch=dev_databatch, path_train=path_train, path_valid=path_valid, path_snapshot=path_snapshot, path_pred=os.path.join(config.getpath("results"), basename + ".valid_pred.ctrees"), path_gold=os.path.join(config.getpath("results"), basename + ".valid_gold.ctrees")) elif actiontype == "evaluate": with chainer.using_config("train", False), chainer.no_backprop_mode(): # Test parsing.parse(model=model, decoder=decoder, databatch=test_databatch, path_pred=path_pred) scores = rst_parseval.evaluate( pred_path=path_pred, gold_path=os.path.join(config.getpath("data"), "rstdt", "renamed", "test.labeled.nary.ctrees")) old_scores = old_rst_parseval.evaluate( pred_path=path_pred, gold_path=os.path.join(config.getpath("data"), "rstdt", "renamed", "test.labeled.nary.ctrees")) out = { "Morey2018": { "Unlabeled Precision": scores["S"]["Precision"] * 100.0, "Precision_info": scores["S"]["Precision_info"], "Unlabeled Recall": scores["S"]["Recall"] * 100.0, "Recall_info": scores["S"]["Recall_info"], "Micro F1": scores["S"]["Micro F1"] * 100.0 }, "Marcu2000": { "Unlabeled Precision": old_scores["S"]["Precision"] * 100.0, "Precision_info": old_scores["S"]["Precision_info"], "Unlabeled Recall": old_scores["S"]["Recall"] * 100.0, "Recall_info": old_scores["S"]["Recall_info"], "Micro F1": old_scores["S"]["Micro F1"] * 100.0 } } utils.write_json(path_eval, out) utils.writelog(utils.pretty_format_dict(out)) utils.writelog("Done: %s" % basename)
def train_model(window_size, max_epochs, patience): root_dir = join('data', 'nets') # the file from which to load pre-trained weights #init_file = join(root_dir, # 'subj%d_weights_deep_nocsp_wide.pickle' % ( # 4)) #init_file = join(root_dir, # 'weights_super_deeper.pickle') init_file = None # the file to which the learned weights will be written weights_file = join(root_dir, 'weights.pickle') temp_weights_file = join(root_dir, 'epoch_%d.pickle') train_data, train_events = [], [] valid_data, valid_events = [], [] for subj_id in range(1, 13): print('loading time series for subject %d...' % (subj_id)) subj_data_list, subj_events_list = utils.load_subject_train(subj_id) print(' creating train and validation sets...') subj_train_data, subj_train_events, subj_valid_data, subj_valid_events = \ utils.split_train_test_data(subj_data_list, subj_events_list, val_size=2, rand=False) train_data += subj_train_data train_events += subj_train_events valid_data += subj_valid_data valid_events += subj_valid_events print('using %d time series for training' % (len(train_data))) print('using %d time series for validation' % (len(valid_data))) print('creating fixed-size time-windows of size %d' % (window_size)) # the training windows should be in random order train_slices = batching.get_permuted_windows(train_data, window_size, rand=True) valid_slices = batching.get_permuted_windows(valid_data, window_size, rand=True) print('there are %d windows for training' % (len(train_slices))) print('there are %d windows for validation' % (len(valid_slices))) #batch_size = 64 batch_size = 512 num_channels = 32 num_actions = 6 train_data, valid_data = \ utils.preprocess(train_data, valid_data) print('building model %s...' % (sys.modules[build_model.__module__].__name__)) l_out = build_model(None, num_channels, window_size, num_actions) all_layers = layers.get_all_layers(l_out) print('this network has %d learnable parameters' % (layers.count_params(l_out))) for layer in all_layers: print('Layer %s has output shape %r' % (layer.name, layer.output_shape)) if init_file is not None: print('loading model weights from %s' % (init_file)) with open(init_file, 'rb') as ifile: src_layers = pickle.load(ifile) dst_layers = layers.get_all_params(l_out) for i, (src_weights, dst_layer) in enumerate(zip(src_layers, dst_layers)): print('loading pretrained weights for %s' % (dst_layer.name)) dst_layer.set_value(src_weights) else: print('all layers will be trained from random initialization') #1r = theano.shared(np.cast['float32'](0.001)) lr = theano.shared(np.cast['float32'](0.01)) mntm = 0.9 print('compiling theano functions...') train_iter = iter_funcs.create_iter_funcs_train(lr, mntm, l_out) valid_iter = iter_funcs.create_iter_funcs_valid(l_out) best_weights = None best_valid_loss = np.inf best_epoch = 0 print('starting training for all subjects at %s' % (utils.get_current_time())) try: for epoch in range(max_epochs): print('epoch: %d' % (epoch)) train_losses, training_outputs, training_inputs = [], [], [] num_batches = (len(train_slices) + batch_size - 1) / batch_size t_train_start = time() for i, (Xb, yb) in enumerate( batching.batch_iterator(batch_size, train_slices, train_data, train_events, window_norm=False)): t_batch_start = time() # hack for faster debugging #if i < 70000: # continue train_loss, train_output = \ train_iter(Xb, yb) if np.isnan(train_loss): print('nan loss encountered in minibatch %d' % (i)) continue train_losses.append(train_loss) assert len(yb) == len(train_output) for input, output in zip(yb, train_output): training_inputs.append(input) training_outputs.append(output) batch_duration = time() - t_batch_start if i % 10 == 0: eta = batch_duration * (num_batches - i) m, s = divmod(eta, 60) h, m = divmod(m, 60) print(' training... (ETA = %d:%02d:%02d)\r' % (h, m, s)), sys.stdout.flush() avg_train_loss = np.mean(train_losses) training_inputs = np.vstack(training_inputs) training_outputs = np.vstack(training_outputs) train_roc = roc_auc_score(training_inputs, training_outputs) train_duration = time() - t_train_start print('') print(' train loss: %.6f' % (avg_train_loss)) print(' train roc: %.6f' % (train_roc)) print(' duration: %.2f s' % (train_duration)) valid_losses, valid_outputs, valid_inputs = [], [], [] num_batches = (len(valid_slices) + batch_size - 1) / batch_size t_valid_start = time() for i, (Xb, yb) in enumerate( batching.batch_iterator(batch_size, valid_slices, valid_data, valid_events, window_norm=False)): t_batch_start = time() valid_loss, valid_output = \ valid_iter(Xb, yb) if np.isnan(valid_loss): print('nan loss encountered in minibatch %d' % (i)) continue valid_losses.append(valid_loss) assert len(yb) == len(valid_output) for input, output in zip(yb, valid_output): valid_inputs.append(input) valid_outputs.append(output) batch_duration = time() - t_batch_start if i % 10 == 0: eta = batch_duration * (num_batches - i) m, s = divmod(eta, 60) h, m = divmod(m, 60) print(' validation... (ETA = %d:%02d:%02d)\r' % (h, m, s)), sys.stdout.flush() # allow training without validation if valid_losses: avg_valid_loss = np.mean(valid_losses) valid_inputs = np.vstack(valid_inputs) valid_outputs = np.vstack(valid_outputs) valid_roc = roc_auc_score(valid_inputs, valid_outputs) valid_duration = time() - t_valid_start print('') print(' valid loss: %.6f' % (avg_valid_loss)) print(' valid roc: %.6f' % (valid_roc)) print(' duration: %.2f s' % (valid_duration)) else: print(' no validation...') # if we are not doing validation we always want the latest weights if not valid_losses: best_epoch = epoch model_train_loss = avg_train_loss model_train_roc = train_roc model_valid_roc = -1. best_valid_loss = -1. best_weights = layers.get_all_param_values(l_out) elif avg_valid_loss < best_valid_loss: best_epoch = epoch model_train_roc = train_roc model_valid_roc = valid_roc model_train_loss = avg_train_loss best_valid_loss = avg_valid_loss best_weights = layers.get_all_param_values(l_out) temp_file = temp_weights_file % (epoch) print('saving temporary best weights to %s' % (temp_file)) with open(temp_file, 'wb') as ofile: pickle.dump(best_weights, ofile, protocol=pickle.HIGHEST_PROTOCOL) if epoch > best_epoch + patience: break best_epoch = epoch new_lr = 0.5 * lr.get_value() lr.set_value(np.cast['float32'](new_lr)) print('setting learning rate to %.6f' % (new_lr)) except KeyboardInterrupt: print('caught Ctrl-C, stopping training...') with open(weights_file, 'wb') as ofile: print('saving best weights to %s' % (weights_file)) pickle.dump(best_weights, ofile, protocol=pickle.HIGHEST_PROTOCOL) print('finished training for all subjects at %s' % (utils.get_current_time())) return model_train_loss, best_valid_loss, model_train_roc, model_valid_roc
def traffic_report_filter(cls, contributor_id): query = cls.query.filter(cls.status == 'published') query = query.join(Disruption) query = query.filter(Disruption.contributor_id == contributor_id) query = query.filter(between(get_current_time(), Disruption.start_publication_date, Disruption.end_publication_date)) return query.all()
from utils import load_data, get_current_time, create_dirs, \ create_minibatches, write_to_tensorboard, \ create_summary_and_projector, create_evaluation_tensor import tensorflow as tf import os # this project uses tensorboard. You can launch tensorboard by executing # "tensorboard --logdir=log" in your project folder # Set parameters learning_rate = 0.001 minibatch_size = 125 num_epochs = 20 latent_space_size = 10 log_dir = "log" current_run = get_current_time() # Create necessary directories log_path, run_path = create_dirs(log_dir, current_run) # Load MNIST data imgs, lbls = load_data() mbs = create_minibatches(imgs, lbls, minibatch_size) # Prepare evaluation set # this set is used to visualize embedding space and decoding results evaluation_set = mbs[0] evaluation_shape = (minibatch_size, latent_space_size) def create_model(input_shape):
help='model type (dcgan | unet | resnet)') parser.add_argument('--data_threads', type=int, default=24, help='number of parallel data loading threads') parser.add_argument('--normalize', action='store_true', help='if true, normalize pose vector') parser.add_argument('--data_type', default='drnet', help='speed up data loading for drnet training') opt = parser.parse_args() opt.dataset = 'kth' rand_str = '@' + str(uuid.uuid4())[:4] name = utils.get_current_time( ) + rand_str + os.sep + 'content_model=%s-pose_model=%s-content_dim=%d-pose_dim=%d-max_step=%d-sd_weight=%.3f-lr=%.3f-sd_nf=%d-normalize=%s' % ( opt.content_model, opt.pose_model, opt.content_dim, opt.pose_dim, opt.max_step, opt.sd_weight, opt.lr, opt.sd_nf, opt.normalize) opt.log_dir = '%s/%s%dx%d/%s' % (opt.log_dir, opt.dataset, opt.image_width, opt.image_width, name) if not os.path.exists(opt.log_dir): os.makedirs('%s/rec/' % opt.log_dir, exist_ok=True) copyfile('./train_drnet.py', opt.log_dir + os.sep + 'train_drnet.py') copyfile('./train_lstm.py', opt.log_dir + os.sep + 'train_lstm.py') # checkpoint = torch.load('%s/model.pth' % opt.log_dir, map_location='cpu') # print('checkpoint: ', checkpoint) has_cuda = torch.cuda.is_available() device = torch.device("cuda" if has_cuda else "cpu")
def store_env_data(gas, temp): with lock: publish('data', {'time': get_current_time(), 'gas': gas, 'temp': temp})
def test_speed(opts): t = utils.get_current_time() shell.call(opts['test']) return utils.get_elapsed_time(t)
model = config.models.get(args.model, None) if model is None: print("Model name not found !!\nvalid names are: {} ".format( config.models)) sys.exit(0) config.global_config['dictionaryfile'] = config.global_config[ 'dictionaryfile'].format(args.dataset) config.global_config['glove'] = config.global_config['glove'].format( args.dataset) model = model(N_classes, **config.global_config) model = model.to(device) savefolder = '_'.join([args.dataset, args.model, args.save]) logger = Logger(os.path.join(savefolder, 'log.txt')) logger.write("==== {} ====".format(get_current_time())) print(model) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) start_epoch = 0 if args.load: start_epoch, meta = load_checkpoint(args.load, model, optimizer) else: logger.write("== {} ==".format(args.expl)) logger.write(str(args).replace(',', ',\n')) #log source code of model being used logger.write_silent(inspect.getsource(type(model))) logger.write_silent(repr(model)) dskwargs = { 'testrun': args.testrun,
def collect_cilium_bugtool_output_per_pod(self, podstatus): bugtool_output_dir = "bugtool-{}-{}".format( podstatus.name, utils.get_current_time()) bugtool_output_file_name = "{}.tar".format(bugtool_output_dir) cmd = "kubectl exec -n {} {} -c cilium-agent cilium-bugtool".format( podstatus.namespace, podstatus.name) try: encoded_output = subprocess.check_output( cmd.split(), shell=False, stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as exc: if exc.returncode != 0: log.error( "Error: {}. Could not run cilium-bugtool on {}" .format(exc, podstatus.name)) else: output = encoded_output.decode() p = re.compile( "^ARCHIVE at (.*)$") output_file_name = "" for line in output.splitlines(): match = p.search(line) if match: output_file_name = match.group(1) if output_file_name == "": log.error( "Error: could not find cilium-bugtool output file name", ) copyCmd = "kubectl cp {}/{}:{} ./{}/{}".format( podstatus.namespace, podstatus.name, output_file_name, self.sysdump_dir_name, bugtool_output_file_name) mkdirCmd = "mkdir -p ./{}/{}".format( self.sysdump_dir_name, bugtool_output_dir) tarCmd = "tar -xf ./{}/{} -C ./{}/{} --strip-components=1".format( self.sysdump_dir_name, bugtool_output_file_name, self.sysdump_dir_name, bugtool_output_dir) rmCmd = "rm ./{}/{}".format( self.sysdump_dir_name, bugtool_output_file_name) try: subprocess.check_output( copyCmd.split(), shell=False, stderr=subprocess.STDOUT, ) subprocess.check_output( mkdirCmd.split(), shell=False, stderr=subprocess.STDOUT, ) subprocess.check_output( tarCmd.split(), shell=False, stderr=subprocess.STDOUT, ) subprocess.check_output( rmCmd.split(), shell=False, stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as exc: if exc.returncode != 0: log.error( "Error: {} Could not collect cilium-bugtool" " output: {}".format( exc, bugtool_output_file_name)) else: log.info("collected cilium-bugtool output: {}".format( bugtool_output_file_name))
def get_account_info(self, recv_window=None): timestamp = get_current_time() parameters = self.craft_parameters(recvWindow=recv_window, timestamp=timestamp) return self.do_get(self.build_url(self.GET_ACCOUNT_INFO_ENDPOINT), self.check_code, parameters)
def commit_log(self, collection, log): log['collection'] = collection log['status'] = '0' log['time'] = utils.get_current_time() self.commit_db['log'].insert(log)
def main(args): #################### # Arguments tree_sampling = args.tree_sampling # NOTE trial_name = args.name # Check assert len(tree_sampling.split("_")) == 3 for type_ in tree_sampling.split("_"): assert type_ in ["X", "BU", "TD", "RB", "LB", "RB2"] assert tree_sampling.split("_")[2] != "X" assert tree_sampling.split("_")[1] != "RB2" assert tree_sampling.split("_")[2] != "RB2" if trial_name is None or trial_name == "None": trial_name = utils.get_current_time() #################### # Path setting config = utils.Config() basename = "%s.%s" \ % (tree_sampling, trial_name) utils.mkdir(os.path.join(config.getpath("results"), "baselines")) path_log = os.path.join(config.getpath("results"), "baselines", basename + ".evaluation.log") path_pred = os.path.join(config.getpath("results"), "baselines", basename + ".evaluation.ctrees") path_eval = os.path.join(config.getpath("results"), "baselines", basename + ".evaluation.json") utils.set_logger(path_log) #################### # Random seed random_seed = trial_name random_seed = utils.hash_string(random_seed) np.random.seed(random_seed) cuda.cupy.random.seed(random_seed) #################### # Log so far utils.writelog("tree_sampling=%s" % tree_sampling) utils.writelog("trial_name=%s" % trial_name) utils.writelog("path_log=%s" % path_log) utils.writelog("path_pred=%s" % path_pred) utils.writelog("path_eval=%s" % path_eval) utils.writelog("random_seed=%d" % random_seed) #################### # Data preparation begin_time = time.time() test_databatch = dataloader.read_rstdt("test", relation_level="coarse-grained", with_root=False) end_time = time.time() utils.writelog("Loaded the corpus. %f [sec.]" % (end_time - begin_time)) #################### # Tree-sampler preparation sampler = treesamplers.TreeSampler(tree_sampling.split("_")) # NOTE with chainer.using_config("train", False), chainer.no_backprop_mode(): parse(sampler=sampler, databatch=test_databatch, path_pred=path_pred) scores = rst_parseval.evaluate( pred_path=path_pred, gold_path=os.path.join(config.getpath("data"), "rstdt", "renamed", "test.labeled.nary.ctrees")) old_scores = old_rst_parseval.evaluate( pred_path=path_pred, gold_path=os.path.join(config.getpath("data"), "rstdt", "renamed", "test.labeled.nary.ctrees")) out = { "Morey2018": { "Unlabeled Precision": scores["S"]["Precision"] * 100.0, "Precision_info": scores["S"]["Precision_info"], "Unlabeled Recall": scores["S"]["Recall"] * 100.0, "Recall_info": scores["S"]["Recall_info"], "Micro F1": scores["S"]["Micro F1"] * 100.0 }, "Marcu2000": { "Unlabeled Precision": old_scores["S"]["Precision"] * 100.0, "Precision_info": old_scores["S"]["Precision_info"], "Unlabeled Recall": old_scores["S"]["Recall"] * 100.0, "Recall_info": old_scores["S"]["Recall_info"], "Micro F1": old_scores["S"]["Micro F1"] * 100.0 } } utils.write_json(path_eval, out) utils.writelog(utils.pretty_format_dict(out)) utils.writelog("Done.")
def all_with_filter(cls, contributor_id, publication_status, tags, uri, statuses): availlable_filters = { 'past': and_(cls.end_publication_date != None, cls.end_publication_date < get_current_time()), 'ongoing': and_( cls.start_publication_date <= get_current_time(), or_(cls.end_publication_date == None, cls.end_publication_date >= get_current_time())), 'coming': Disruption.start_publication_date > get_current_time() } query = cls.query.filter( and_(cls.contributor_id == contributor_id, cls.status.in_(statuses))) if tags: query = query.filter(cls.tags.any(Tag.id.in_(tags))) if uri: query = query.join(cls.impacts) query = query.filter(Impact.status == 'published') query = query.join(Impact.objects) #Here add a new query to find impacts with line =_section having uri as line, start_point or end_point filters = [] alias_line = aliased(PTobject) alias_start_point = aliased(PTobject) alias_end_point = aliased(PTobject) alias_route = aliased(PTobject) alias_via = aliased(PTobject) query_line_section = query query_line_section = query_line_section.join(PTobject.line_section) query_line_section = query_line_section.join( alias_line, LineSection.line_object_id == alias_line.id) filters.append(alias_line.uri == uri) query_line_section = query_line_section.join( PTobject, LineSection.object_id == PTobject.id) query_line_section = query_line_section.join( alias_start_point, LineSection.start_object_id == alias_start_point.id) filters.append(alias_start_point.uri == uri) query_line_section = query_line_section.join( alias_end_point, LineSection.end_object_id == alias_end_point.id) filters.append(alias_end_point.uri == uri) query_line_section = query_line_section.join( alias_route, LineSection.routes) filters.append(alias_route.uri == uri) query_line_section = query_line_section.join( alias_via, LineSection.via) filters.append(alias_via.uri == uri) query_line_section = query_line_section.filter(or_(*filters)) query = query.filter(PTobject.uri == uri) publication_status = set(publication_status) if len(publication_status) == len(publication_status_values): #For a query by uri use union with the query for line_section if uri: query = query.union_all(query_line_section) else: filters = [ availlable_filters[status] for status in publication_status ] query = query.filter(or_(*filters)) #For a query by uri use union with the query for line_section if uri: query_line_section = query_line_section.filter(or_(*filters)) query = query.union_all(query_line_section) return query.order_by(cls.end_publication_date, cls.id)
def traffic_report_filter(cls, contributor_id): query = cls.query.filter(cls.status == 'published') query = query.filter(cls.contributor_id == contributor_id) query = query.filter( between(get_current_time(), cls.start_publication_date, cls.end_publication_date)) return query.all()
def main(cfg): start_time = utils.get_current_time() # override base-config parameters with arguments provided at run-time base_cfg_dict = utils.load_json(cfg.base_config) membership = cutils.get_membership(base_cfg_dict) cfg_dict = vars(cfg) cfg_dict = { key: cfg_dict[key] for key in cfg_dict if cfg_dict[key] is not None } updated_cfg_dict = cutils.update_params(base_cfg_dict, cfg_dict, membership) cfg = Namespace(**updated_cfg_dict) utils.make_dirs('./config/save/', replace=False) utils.save_json(updated_cfg_dict, './config/save/config_{}.json'.format(start_time)) cfg.time = start_time cfg.model_name = '{}_{}'.format(cfg.model_name, start_time) # setting up output directories, and writing to stdout utils.make_dirs(os.path.join(cfg.stdout_dir, cfg.model_type), replace=False) sys.stdout = open( r'./{}/{}/stdout_{}_{}.txt'.format(cfg.stdout_dir, cfg.model_type, cfg.model_type, cfg.model_name), 'w') print(cfg) utils.flush() if cfg.plot: utils.make_dirs(os.path.join(cfg.plot_dir, cfg.model_type, cfg.model_name), replace=True) if cfg.save_model: utils.make_dirs(os.path.join(cfg.model_dir, cfg.model_type, cfg.model_name), replace=True) # set random seed if cfg.random_seed == 0: cfg.random_seed = random.randint(1, 10000) print('random seed set to {}'.format(cfg.random_seed)) utils.flush() random.seed(cfg.random_seed) np.random.seed(cfg.random_seed) torch.manual_seed(cfg.random_seed) # set device as cuda or cpu if cfg.device.lower() == 'cuda' and torch.cuda.is_available(): # reproducibility using cuda torch.cuda.manual_seed(cfg.random_seed) cudnn.deterministic = True cudnn.benchmark = False else: if cfg.device.lower() == 'cuda': print( 'device option was set to <cuda>, but no cuda device was found' ) utils.flush() cfg.device = 'cpu' trainer = Trainer(cfg) trainer.train()
def now(): return str(get_current_time())
def main(args): os.environ["CUDA_VISIBLE_DEVICES"] = \ ','.join(str(gpu) for gpu in args.visible_gpus) device = "cuda" if torch.cuda.is_available() else "cpu" dataset = args.dataset net_name = args.cnn_name root_path = '.' data_path = os.path.join(root_path, "data") save_path = os.path.join(root_path, "results", "iDLG_%s_%s" % (dataset, net_name)) if args.add_clamp: save_path += "_clamp" # Some running setting initial_lr = args.lr num_dummy = 1 Iteration = args.max_iter plot_steps = args.plot_steps # run_methods = ["iDLG", "DLG"] run_methods = ["iDLG"] tt = transforms.Compose([transforms.ToTensor()]) tp = transforms.Compose([transforms.ToPILImage()]) print(dataset, 'root_path:', root_path) print(dataset, 'data_path:', data_path) print(dataset, 'save_path:', save_path) if not os.path.exists('results'): os.mkdir('results') if not os.path.exists(save_path): os.mkdir(save_path) """ load data """ data_params = DatasetParams() data_params.config(name=dataset, root_path=root_path, data_path=data_path) shape_img = data_params.shape_img num_classes = data_params.num_classes channel = data_params.channel dst = data_params.dst selected_indices = data_params.selected_indices cmap = data_params.cmap # Build ConvNet net = config_net(net_name=net_name, input_shape=(channel, ) + shape_img, num_classes=num_classes) net = net.to(device) # net.eval() # Load model pretrain weights if os.path.isfile(args.model_ckpt): ckpt = torch.load(args.model_ckpt) net.load_state_dict(ckpt) num_success = 0 num_exp = len(selected_indices) criterion = nn.CrossEntropyLoss().to(device) ''' train DLG and iDLG ''' for idx_exp in range(num_exp): print('running %d|%d experiment' % (idx_exp, num_exp)) np.random.seed(idx_exp) # idx_shuffle = np.random.permutation(len(dst)) for method in run_methods: print('%s, Try to generate %d images' % (method, num_dummy)) # criterion = nn.CrossEntropyLoss().to(device) imidx_list = [] # get ground truth image and label idx = selected_indices[idx_exp] imidx_list.append(idx) tmp_datum = tt(dst[idx][0]).float().to(device) tmp_datum = tmp_datum.view(1, *tmp_datum.size()) tmp_label = torch.Tensor([dst[idx][1]]).long().to(device) tmp_label = tmp_label.view(1, ) gt_data = tmp_datum gt_label = tmp_label # compute original gradient out = net(gt_data) y = criterion(out, gt_label) dy_dx = torch.autograd.grad(y, net.parameters()) orig_dy_dx = list((t.detach().clone() for t in dy_dx)) if args.grad_norm: grad_max = [x.max().item() for x in orig_dy_dx] grad_min = [x.min().item() for x in orig_dy_dx] orig_dy_dx = [ (g - g_min) / (g_max - g_min) for g, g_min, g_max in zip(orig_dy_dx, grad_min, grad_max) ] # generate dummy data and label torch.manual_seed(10) dummy_data = torch.randn( gt_data.size()).to(device).requires_grad_(True) dummy_label = torch.randn( (gt_data.shape[0], num_classes)).to(device).requires_grad_(True) # truncated dummy image and label if args.add_clamp: dummy_data.data = torch.clamp(dummy_data + 0.5, 0, 1) dummy_label.data = torch.clamp(dummy_label + 0.5, 0, 1) if method == 'DLG': # optim_obj = [dummy_data, dummy_label] optimizer = torch.optim.LBFGS( [{ 'params': [dummy_data, dummy_label], 'initial_lr': initial_lr }], lr=initial_lr, max_iter=50, tolerance_grad=1e-9, tolerance_change=1e-11, history_size=250, line_search_fn="strong_wolfe") elif method == 'iDLG': # optim_obj = [dummy_data, ] optimizer = torch.optim.LBFGS([{ 'params': [dummy_data], 'initial_lr': initial_lr }], lr=initial_lr, max_iter=50, tolerance_grad=1e-9, tolerance_change=1e-11, history_size=250, line_search_fn="strong_wolfe") # predict the ground-truth label label_pred = torch.argmin(torch.sum(orig_dy_dx[-2], dim=-1), dim=-1).detach().reshape( (1, )).requires_grad_(False) history = [] history_iters = [] losses = [] mses = [] train_iters = [] scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.95, last_epoch=-1) print('lr =', initial_lr) for iters in range(Iteration): closure = functools.partial(_closure, optimizer=optimizer, dummy_data=dummy_data, dummy_label=dummy_label, label_pred=label_pred, method=method, criterion=criterion, net=net, orig_dy_dx=orig_dy_dx, grad_norm=args.grad_norm) optimizer.step(closure) # pixel value clamp if args.add_clamp: dummy_data.data = torch.clamp(dummy_data, 0, 1) current_loss = closure().item() train_iters.append(iters) losses.append(current_loss) mses.append(torch.mean((dummy_data - gt_data)**2).item()) scheduler.step() if iters % plot_steps == 0: current_time = get_current_time() print(current_time, iters, 'loss = %.8f, mse = %.8f' % (current_loss, mses[-1])) history.append([ tp(dummy_data[imidx].cpu()) for imidx in range(num_dummy) ]) history_iters.append(iters) for imidx in range(num_dummy): plot_dummy_x(imidx, cmap, tp, gt_data, history, history_iters, save_path, method, selected_indices, idx_exp) # if current_loss < 0.000001: # converge if mses[-1] < 1e-4: break if mses[-1] < 1e-3: num_success += 1 # Save mse curve plot_mse_curve(mses, iters, save_path, method, selected_indices, idx_exp) if method == 'DLG': loss_DLG = losses label_DLG = torch.argmax(dummy_label, dim=-1).detach().item() mse_DLG = mses elif method == 'iDLG': loss_iDLG = losses label_iDLG = label_pred.item() mse_iDLG = mses print('gt_label:', gt_label.detach().cpu().data.numpy()) if "DLG" in run_methods: print('loss_DLG:', loss_DLG[-1], 'mse_DLG:', mse_DLG[-1], 'lab_DLG:', label_DLG) if "iDLG" in run_methods: print('loss_iDLG:', loss_iDLG[-1], 'mse_iDLG:', mse_iDLG[-1], 'lab_iDLG:', label_iDLG) print('----------------------\n\n') print("Number of successful recover:", num_success)
def add_page(self, page_number, items): """ Adds Items to be cached """ self._cached_items[page_number] = items self._cached_at = utils.get_current_time()
def addItem(self, status: str): self.table.insertRow(0) self.table.setItem(0, 0, QTableWidgetItem(get_current_time())) self.table.setItem(0, 1, QTableWidgetItem(status))