def all_with_filter(cls, contributor_id, publication_status, tags, uri,
statuses):
availlable_filters = {
'past': and_(cls.end_publication_date != None, cls.end_publication_date < get_current_time()),
'ongoing': and_(cls.start_publication_date <= get_current_time(),
or_(cls.end_publication_date == None, cls.end_publication_date >= get_current_time())),
'coming': Disruption.start_publication_date > get_current_time()
}
query = cls.query.filter(and_(
cls.contributor_id == contributor_id,
cls.status.in_(statuses)
))
if tags:
query = query.filter(cls.tags.any(Tag.id.in_(tags)))
if uri:
query = query.join(cls.impacts)
query = query.filter(Impact.status == 'published')
query = query.join(Impact.objects)
#Here add a new query to find impacts with line =_section having uri as line, start_point or end_point
filters = []
alias_line = aliased(PTobject)
alias_start_point = aliased(PTobject)
alias_end_point = aliased(PTobject)
alias_route = aliased(PTobject)
alias_via = aliased(PTobject)
query_line_section = query
query_line_section = query_line_section.join(PTobject.line_section)
query_line_section = query_line_section.join(alias_line, LineSection.line_object_id == alias_line.id)
filters.append(alias_line.uri == uri)
query_line_section = query_line_section.join(PTobject, LineSection.object_id == PTobject.id)
query_line_section = query_line_section.join(alias_start_point, LineSection.start_object_id == alias_start_point.id)
filters.append(alias_start_point.uri == uri)
query_line_section = query_line_section.join(alias_end_point, LineSection.end_object_id == alias_end_point.id)
filters.append(alias_end_point.uri == uri)
query_line_section = query_line_section.join(alias_route, LineSection.routes)
filters.append(alias_route.uri == uri)
query_line_section = query_line_section.join(alias_via, LineSection.via)
filters.append(alias_via.uri == uri)
query_line_section = query_line_section.filter(or_(*filters))
query = query.filter(PTobject.uri == uri)
publication_status = set(publication_status)
if len(publication_status) == len(publication_status_values):
#For a query by uri use union with the query for line_section
if uri:
query = query.union_all(query_line_section)
else:
filters = [availlable_filters[status] for status in publication_status]
query = query.filter(or_(*filters))
#For a query by uri use union with the query for line_section
if uri:
query_line_section = query_line_section.filter(or_(*filters))
query = query.union_all(query_line_section)
return query.order_by(cls.end_publication_date)
def all_with_filter(cls, contributor_id, publication_status, ends_after_date, ends_before_date, tags, uri, line_section, statuses):
availlable_filters = {
'past': and_(cls.end_publication_date != None, cls.end_publication_date < get_current_time()),
'ongoing': and_(cls.start_publication_date <= get_current_time(),
or_(cls.end_publication_date == None, cls.end_publication_date >= get_current_time())),
'coming': Disruption.start_publication_date > get_current_time()
}
query = cls.query.filter(and_(
cls.contributor_id == contributor_id,
cls.status.in_(statuses)
))
if ends_after_date:
query = query.filter(cls.end_publication_date >= ends_after_date)
if ends_before_date:
query = query.filter(cls.end_publication_date <= ends_before_date)
if tags:
query = query.filter(cls.tags.any(Tag.id.in_(tags)))
if uri:
query = query.join(cls.impacts)
query = query.filter(Impact.status == 'published')
query = query.join(Impact.objects)
#Here add a new query to find impacts with line_section having uri as line
if line_section :
query_line_section = query
query_line_section = query_line_section.filter(and_(PTobject.type == "line_section", PTobject.uri.like(uri + ":%")))
query = query.filter(PTobject.uri == uri)
publication_status = set(publication_status)
if len(publication_status) == len(publication_status_values):
#For a query by uri use union with the query for line_section
if uri and line_section:
query = query.union(query_line_section)
else:
filters = [availlable_filters[status] for status in publication_status]
query = query.filter(or_(*filters))
#For a query by uri use union with the query for line_section
if uri and line_section:
query_line_section = query_line_section.filter(or_(*filters))
query = query.union(query_line_section)
return query.order_by(cls.end_publication_date, cls.id)
def populate_db():
with open('data/states.csv', mode='r') as infile:
reader = csv.reader(infile)
mydict = {rows[0]: {'pledged':rows[1], 'unpledged':rows[2], 'date': rows[4]} for rows in reader}
for state, delegates in mydict.items():
s = models.State(state=state)
s.pledged_available = int(delegates['pledged'])
s.unpledged_available = int(delegates['unpledged'])
try:
s.election_date = datetime.strptime(delegates['date'], '%m/%d/%Y')
except ValueError as e:
print 439750983246709479074092760974092764097460927903467
print delegates['date']
s.total_available = s.pledged_available + s.unpledged_available
s.clinton_percentage = 0
s.sanders_percentage = 0
s.clinton_delegates = 0
s.sanders_delegates = 0
s.last_updated = get_current_time()
db.session.add(s)
db.session.commit()
clinton = models.Candidate(last_name='Clinton', first_name='Hillary')
sanders = models.Candidate(last_name='Sanders', first_name='Bernie')
db.session.add(clinton)
db.session.add(sanders)
db.session.commit()
get_charts()
get_results()
def get_results():
with open('data/results.csv', mode='r') as infile:
reader = csv.reader(infile)
mydict = {rows[0]: {'clinton_pledged':rows[1], 'clinton_unpledged':rows[2], 'sanders_pledged':rows[3], 'sanders_unpledged':rows[4], 'clinton_percentage':rows[5], 'sanders_percentage':rows[6]} for rows in reader}
clinton = models.Candidate.query.filter_by(last_name='Clinton').first()
sanders = models.Candidate.query.filter_by(last_name='Sanders').first()
for state, results in mydict.items():
if models.State.query.filter_by(state=state).first():
s = models.State.query.filter_by(state=state).first()
s.last_updated = get_current_time()
s.clinton_pledged_delegates_results = int(results['clinton_pledged'])
s.sanders_pledged_delegates_results = int(results['sanders_pledged'])
s.clinton_unpledged_delegates_results = int(results['clinton_unpledged'])
s.sanders_unpledged_delegates_results = int(results['sanders_unpledged'])
db.session.add(s)
s.clinton_percentage_results = float(results['clinton_percentage'])
s.sanders_percentage_results = float(results['sanders_percentage'])
clinton.pledged_delegates += int(results['clinton_pledged'])
clinton.unpledged_delegates += int(results['clinton_unpledged'])
sanders.pledged_delegates += int(results['sanders_pledged'])
sanders.unpledged_delegates += int(results['sanders_unpledged'])
clinton.total_delegates = clinton.pledged_delegates + clinton.unpledged_delegates
sanders.total_delegates = sanders.pledged_delegates + sanders.unpledged_delegates
db.session.commit()
def traffic_report_filter(cls, contributor_id):
query = cls.query.filter(cls.status == "published")
query = query.join(Disruption)
query = query.filter(Disruption.contributor_id == contributor_id)
query = query.filter(
between(get_current_time(), Disruption.start_publication_date, Disruption.end_publication_date)
)
return query.all()
def collection_qa_analyzer(col_name, env='active', fields=None):
# parse configuration file
cfg_fpath = os.path.join(basepath, "db_and_fpath.cfg")
parser = ConfigParser()
parser.read(cfg_fpath)
# get detailed info from configuration
mongo_uri = parser.get(env, 'uri')
db_name = parser.get(env, 'db_name')
io_fpath = parser.get(env, 'io_fpath')
# get current time in string
current_time = get_current_time(flag='short')
# create directories if not existing, and return the collection's path
col_fpath = make_dirs(io_fpath, current_time, db_name, col_name)
# get all file paths that are required
basic_stat_fpath = os.path.join(col_fpath, 'basic_stat.json')
domain_fpath = os.path.join(col_fpath, 'domain.json')
existing_checking_fpath = os.path.join(col_fpath, 'existing.json')
field_fpath = os.path.join(col_fpath, 'fields.json')
type_checking_fpath = os.path.join(col_fpath, 'type_checking.json')
nullable_checking_fpath = os.path.join(col_fpath, 'nullable.json')
empty_checking_fpath = os.path.join(col_fpath, 'empty.json')
coverage_fpath = os.path.join(col_fpath, 'coverage.json')
merger_fpath = os.path.join(col_fpath, 'merger.txt')
# MongoEye
mongoEye = MongoEye(mongo_uri, db_name, col_name)
# get detailed STAT
mongoEye.save_existing_checking(existing_checking_fpath)
mongoEye.save_fields(existing_checking_fpath, field_fpath)
mongoEye.save_type_checking(type_checking_fpath, field_fpath)
mongoEye.save_nullable_checking(field_fpath, nullable_checking_fpath)
mongoEye.save_empty_checking(type_checking_fpath, empty_checking_fpath)
mongoEye.save_coverage(existing_checking_fpath,
nullable_checking_fpath,
empty_checking_fpath,
coverage_fpath)
mongoEye.merger(field_fpath, type_checking_fpath,
existing_checking_fpath, nullable_checking_fpath,
empty_checking_fpath, coverage_fpath, merger_fpath)
# get basic STAT
mongoEye.save_domain_counters(domain_fpath)
mongoEye.get_basic_stat(merger_fpath, basic_stat_fpath, domain_fpath)
# test count unique domains
#print mongoEye.count_domains()
#===============================================================================
# 获得某一个字段值的分布(distribution)
#===============================================================================
if fields:
field_dist_collector(mongoEye, fields, col_fpath)
def output(self, key, obj):
current_datetime = get_current_time()
is_future = False
for application_period in obj.application_periods:
if current_datetime >= application_period.start_date and current_datetime <= application_period.end_date:
return 'active'
if current_datetime <= application_period.start_date:
is_future = True
if is_future:
return 'future'
return 'past'
def my_predictions():
is_logged_in, user = is_user_logged_in(session)
if not is_logged_in:
return redirect(url_for("index"))
user_info = user.to_json()
return render_template(
"my-predictions.html",
user=user_info,
fixtures=football_api_client.get_all_fixtures(),
points=get_points_for_user(user_info["predictions"]),
current_time=get_current_time()
)
def profile():
form = ProfileForm(obj=current_user)
if form.validate_on_submit():
form.populate_obj(user)
user.update_at = get_current_time()
db.session.commit()
flash('Public profile updated.', 'success')
return render_template('user/profile.html', form=form)
def save_db(temp):
timestamp = int(time.time())
datetime = get_current_time()
pushed = 0
query = 'insert into measure values (null, %s, "%s", "%s", %s, %s)' % (timestamp, datetime, ROOM, pushed, temp)
try:
conn = sqlite3.connect(get_abs_path() + '/pi-temp.db')
conn.execute(query)
conn.commit()
conn.close()
except:
logging.error('Error when saving measure in database')
pass
def get_context(self):
# get the timezone from the GET params (default to UTC)
tz = self.request.get('tz', default_value='UTC')
# attempt to parse timezone
try:
timezone = get_timezone(tz)
# return error if invalid timezone
except pytz.UnknownTimeZoneError:
# set HTTP status and return error msg
self.response.set_status(400)
return {'error': 'Unknown timezone "%s"' % tz}
# add timezone name to response
response = {'tz_name': timezone.zone}
# add utc offset to response
response['tz_offset'] = get_utcoffset_as_string(timezone)
# add current time to response
response['datetime'] = get_current_time(tz)
return response
def insert_or_update_post(self, post):
if post is None:
return
author = get_post_author_id(post['author'])
board = post['board']
post_time = get_post_time(post['date'])
title = post['article_title']
ip = post['ip']
messages = post['messages']
post_id = post['article_id']
content = post['content']
url = post['url']
self.add_user(author)
self.add_board(board)
with self.__post_insertion_lock:
self.__execute_write(
'''
INSERT OR REPLACE INTO `posts`
(
`id`,
`board`,
`post_id`,
`author`,
`date_time`,
`title`,
`web_url`,
`ip`
)
VALUES
(
(
SELECT `id` FROM `posts`
WHERE `board` = (
SELECT `id` FROM `boards`
WHERE `name` = :board
) AND `post_id` = :post_id
),
(
SELECT `id` FROM `boards`
WHERE `name` = :board
),
:post_id,
(
SELECT `id` FROM `users`
WHERE `username` = :author
),
:date_time,
:title,
:web_url,
:ip
);
''', {
'board': board,
'post_id': post_id,
'author': author,
'date_time': post_time,
'title': title,
'web_url': url,
'ip': ip
})
self.__execute_write(
'''
INSERT OR REPLACE INTO `posts_content`
(`id`, `post`, `content`)
VALUES
(
(
SELECT `id` FROM `posts_content`
WHERE `post` = (
SELECT `id` FROM `posts`
WHERE `board` = (
SELECT `id` FROM `boards`
WHERE `name` = :board
) AND `post_id` = :post_id
)
),
(
SELECT `id` FROM `posts`
WHERE `board` = (
SELECT `id` FROM `boards`
WHERE `name` = :board
) AND `post_id` = :post_id
),
:content
);
''', {
'board': board,
'post_id': post_id,
'content': content
})
# delete existing pushes
self.__execute_write(
'''
DELETE FROM `pushes`
WHERE `post` = (
SELECT `id` FROM `posts`
WHERE `board` = (
SELECT `id` FROM `boards`
WHERE `name` = :board
) AND `post_id` = :post_id
);
''', {
'board': board,
'post_id': post_id
})
year = get_post_year(post['date'])
if year == 0:
return
for push in messages:
author = push['push_userid']
push_content = push['push_content']
push_tag = push['push_tag']
ip, push_time = get_push_ip_time(year, push['push_ipdatetime'])
if push_time < post_time:
_, push_time = get_push_ip_time(year + 1,
push['push_ipdatetime'])
self.add_user(author)
self.__execute_write(
'''
INSERT INTO `pushes`
(
`post`,
`type`,
`author`,
`content`,
`ip`,
`date_time`
)
VALUES
(
(
SELECT `id` FROM `posts`
WHERE `board` = (
SELECT `id` FROM `boards`
WHERE `name` = :board
) AND `post_id` = :post_id
),
:type ,
(
SELECT `id` FROM `users`
WHERE `username` = :author
),
:content ,
:ip ,
:date_time
)
''', {
'board': board,
'post_id': post_id,
'type': push_tag,
'author': author,
'content': push_content,
'ip': ip,
'date_time': push_time
})
self.__execute_write(
'''
INSERT OR REPLACE INTO `crawled_posts`
(`id`, `post`, `date_time`)
VALUES
(
(
SELECT `id` FROM `crawled_posts`
WHERE `post` = (
SELECT `id` FROM `posts`
WHERE `board` = (
SELECT `id` FROM `boards`
WHERE `name` = :board
) AND `post_id` = :post_id
)
),
(
SELECT `id` FROM `posts`
WHERE `board` = (
SELECT `id` FROM `boards`
WHERE `name` = :board
) AND `post_id` = :post_id
),
:date_time
);
''', {
'board': board,
'post_id': post_id,
'date_time': get_current_time()
})
def get_time_left(self, current_datetime=utils.get_current_time()):
return (self.expiry - current_datetime).total_seconds() / (day * 365)
def listen_print_loop(self, responses):
"""Iterates through server responses and prints them.
The responses passed is a generator that will block until a response
is provided by the server.
Each response may contain multiple results, and each result may contain
multiple alternatives; for details, see https://goo.gl/tjCPAU. Here we
print only the transcription for the top alternative of the top result.
In this case, responses are provided for interim results as well. If the
response is an interim one, print a line feed at the end of it, to allow
the next result to overwrite it, until the response is a final one. For the
final one, print a newline to preserve the finalized transcription.
"""
info("Entered listen_print_loop")
for response in responses:
if (get_current_time() - self.stream.start_time >
self.streaming_limit):
self.stream.start_time = get_current_time()
break
if not response.results:
continue
# The `results` list is consecutive. For streaming, we only care about
# the first result being considered, since once it's `is_final`, it
# moves on to considering the next utterance.
result = response.results[0]
if not result.alternatives:
continue
# Display the transcription of the top alternative.
transcript = result.alternatives[0].transcript
result_seconds = 0
result_micros = 0
if result.result_end_time.seconds:
result_seconds = result.result_end_time.seconds
if result.result_end_time.microseconds:
result_micros = result.result_end_time.microseconds
self.stream.result_end_time = int((result_seconds * 1000) +
(result_micros / 1000))
corrected_time = (
self.stream.result_end_time - self.stream.bridging_offset +
(self.streaming_limit * self.stream.restart_counter))
if self.websocket is not None:
message = json.dumps({
"transcript": transcript,
"is_final": result.is_final
})
asyncio.run(self.publish_to_websocket(message))
self.publish_transcript(transcript, result.is_final, "Google")
if result.is_final:
self.stream.is_final_end_time = self.stream.result_end_time
self.stream.last_transcript_was_final = result.is_final
parser.add_argument('--dataset',
default='coco',
choices=['tejani', 'coco'],
type=str,
help="Dataset name")
parser.add_argument('--checkpoint',
default='0.0',
type=str,
help="Checkpoint name in format: `epoch.iteration`")
parser.add_argument('--gpu', default='0', type=str, help="GPU id")
return parser.parse_args()
args = parse_arg()
cfg = config.network[args.dataset]['cfg']
log_dir = opj(config.LOG_ROOT, get_current_time())
writer = SummaryWriter(log_dir=log_dir)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
def train(epoch, trainloader, yolo, optimizer):
"""Training wrapper
@Args
epoch: (int) training epoch
trainloader: (Dataloader) train data loader
yolo: (nn.Module) YOLOv3 model
optimizer: (optim) optimizer
"""
yolo.train()
tbar = tqdm(trainloader, ncols=80)
def collect_logs_per_pod(self, podstatus):
containers = utils.get_container_names_per_pod(podstatus.namespace,
podstatus.name)
for container in containers:
log_file_name = "{}-{}-{}".format(podstatus.name, container,
utils.get_current_time())
command = "kubectl logs {} --container={} --timestamps=true " \
"--since={} --limit-bytes={} -n {} > {}/{}.log"
cmd = command.format(
podstatus.name, container, self.since, self.size_limit,
podstatus.namespace, self.sysdump_dir_name, log_file_name)
try:
subprocess.check_output(
cmd, shell=True, stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as exc:
if exc.returncode != 0:
log.error("Error: {}. Could not collect log file: {}"
.format(exc, log_file_name))
else:
log.info("collected log file: {}".format(log_file_name))
# Only get logs from previous containers if pod has restarted.
# Need to get the pod to access its restartCount.
podCommandPreFormatted = "kubectl get pod {} -n {} -o json"
podCmd = podCommandPreFormatted.format(
podstatus.name, podstatus.namespace,
)
try:
podOutput = subprocess.check_output(
podCmd, shell=True, stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as exc:
if exc.returncode != 0:
log.debug("Debug {}: could not get pod {}").format(
exc, podstatus[0])
else:
# Examine JSON output to see if restartCount > 0
decodedPodOutput = podOutput.decode()
jsonOutput = json.loads(decodedPodOutput)
containerStatuses = jsonOutput['status']['containerStatuses']
gatherPrevLogs = False
for value in containerStatuses:
restartCount = value['restartCount']
if int(restartCount) > 0:
gatherPrevLogs = True
break
if gatherPrevLogs:
log_file_name_previous = "{0}-previous".format(
log_file_name)
command = "kubectl logs --previous --timestamps=true " \
"--container={} " \
"--since={} " \
"--limit-bytes={} -n {} {} > {}/{}.log"
cmd = command.format(container, self.since,
self.size_limit,
podstatus.namespace, podstatus.name,
self.sysdump_dir_name,
log_file_name_previous)
try:
subprocess.check_output(
cmd, shell=True, stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as exc:
if exc.returncode != 0:
log.debug(
"Debug: {}. Could not collect previous "
"log for '{}': {}"
.format(exc, podstatus.name, log_file_name))
else:
log.info("collected log file: {}".format(
log_file_name_previous))
else:
log.debug("no previous pod logs to gather for "
"pod/container {}/{}".format(
podstatus.name, container))
if args.epochs:
SETTINGS['EPOCHS'] = args.epochs
if args.mini == "True":
SETTINGS['NUM_CLASSES'] = 11
SETTINGS['DATA_PATHS']['TRAIN_CSV'] = "mini_train.csv"
SETTINGS['DATA_PATHS']['TEST_CSV'] = "mini_test.csv"
if args.lr:
SETTINGS["LR"] = args.lr
if args.decay:
SETTINGS["DECAY"] = args.decay
SETTINGS["WLOSS"] = args.wloss == "True"
SETTINGS["TRANSFORMER"] = args.transformer
if args.batch:
SETTINGS["BATCH_SIZE"] = args.batch
TIME = get_current_time()
# Load and transform data
transform = get_transformers()[SETTINGS["TRANSFORMER"]]
dataset = Loader(SETTINGS['DATA_PATHS']['TRAIN_CSV'],
SETTINGS['DATA_PATHS']['DATASET_PATH'],
transform=transform)
test_dataset = Loader(SETTINGS['DATA_PATHS']['TEST_CSV'],
SETTINGS['DATA_PATHS']['DATASET_PATH'],
transform=transform)
# Train k models and keep the best
logging.info("Settings: {}".format(str(SETTINGS)))
best_model = train(dataset)
plot_loss(best_model, SETTINGS)
test(best_model, test_dataset)
def train(config):
# load data
user_pos_train, user_pos_test = load_data(config)
all_users = list(user_pos_train.keys())
all_users.sort()
user_num = config['user_num']
item_num = config['item_num']
if not os.path.exists(config['output_dir']):
os.mkdir(config['output_dir'])
with open(os.path.join(config['output_dir'], 'config.json'), 'w') as fout:
print(json.dumps(config), file=fout)
train_log = open(os.path.join(config['output_dir'], 'train_log.txt'), 'w')
# build model
generator = GEN(config)
discriminator = DIS(config)
saver = tf.compat.v1.train.Saver(max_to_keep=1)
model_path = os.path.join(config['output_dir'], 'model/SD-GAR')
os.environ["CUDA_VISIBLE_DEVICES"] = config['CUDA_VISIBLE_DEVICES']
config_tf = tf.compat.v1.ConfigProto()
config_tf.gpu_options.allow_growth = True
sess = tf.compat.v1.Session(config=config_tf)
# init variable
sess.run(tf.compat.v1.global_variables_initializer())
# init embeddings
print("%s; initializing embeddings and constructing alias table..." %
get_current_time())
prob_users, prob_items = get_init_embeddings(config)
alias_table = AliasTable(prob_users, prob_items)
sampler_d = SamplerD(config,
alias_table=alias_table,
generator=generator,
sess=sess)
print("%s; finished" % get_current_time())
# minimax training
best, best_gen = 0., 0.
global Train_d_cnt, Gen_data_for_d_time, Train_d_time
global Train_g_cnt, Gen_data_for_g_time, Train_g_time
mretic_name = [
'P@3', 'P@5', 'P@10', 'P@50', 'NDCG@3', 'NDCG@5', 'NDCG@10', 'NDCG@50',
'MRR'
]
for epoch in range(76):
# train discriminator
if epoch > 0:
Train_d_cnt += 1
batch_num = 0
# the loss incorporates four parts including total loss, positive sample loss, negtive sample loss and regularization
loss_arr = np.array([0.] * 4)
time_start = time.time()
sampler_d.generate_data(user_pos_train,
config['dis_sample_num'],
shuffle=True)
sampler_d.generate_neg_scores()
Gen_data_for_d_time += time.time() - time_start
data_len = len(sampler_d.data)
time_start = time.time()
index = 0
while index < data_len:
batch = sampler_d.get_next_batch(config['batch_size'])
users = batch['users']
pos_items = batch['pos_items']
neg_items = batch['neg_items']
neg_scores = batch['neg_scores']
index += config['batch_size']
_, batch_loss_list = sess.run(
[discriminator.get_update(),
discriminator.get_loss()],
feed_dict={
discriminator.u: users,
discriminator.pos_i: pos_items,
discriminator.neg_i: neg_items,
discriminator.g_scores: neg_scores
})
batch_loss_list[1] = np.mean(batch_loss_list[1])
batch_loss_list[2] = np.mean(batch_loss_list[2])
batch_loss_arr = np.array(batch_loss_list)
loss_arr += batch_loss_arr
batch_num += 1
Train_d_time += time.time() - time_start
loss_arr = loss_arr / batch_num
curr_time = get_current_time()
buf = "%s; epoch: %s; loss: %s; pos_loss: %s; neg_loss: %s; regular_loss: %s" % (
curr_time, epoch, loss_arr[0], loss_arr[1], loss_arr[2],
loss_arr[3])
output_to_file(buf, train_log)
if epoch % 5 == 0:
result = eval(sess, discriminator, user_pos_train,
user_pos_test)
curr_time = get_current_time()
buf = "\t%s; metrics: \t%s" % (curr_time, '\t'.join(
["%7s" % x for x in mretic_name]))
output_to_file(buf, train_log)
buf = "\t%s; performance:\t%s" % (curr_time, '\t'.join(
["%.5f" % x for x in result]))
output_to_file(buf, train_log)
ndcg_50 = result[7]
if ndcg_50 > best:
buf = '\tbest ndcg@50, saving the current model'
output_to_file(buf, train_log)
best = ndcg_50
saver.save(sess, model_path)
f_gen_embeddings = open(
os.path.join(config['output_dir'],
'gen_embeddings.txt'), 'wb')
pickle.dump(
[alias_table.prob_users, alias_table.prob_items],
f_gen_embeddings)
if epoch % 5 == 0:
Train_g_cnt += 1
print("%s; computing partition function..." % get_current_time())
z_u, d_logits = get_partition_funciton(sess, discriminator,
generator, alias_table,
config)
# update user embeddings
print("%s; computing u..." % get_current_time())
prob_users = get_new_user_embeddings(sess, discriminator,
generator, z_u, alias_table,
config)
print("%s; update alias table u..." % get_current_time())
time_start = time.time()
# update user alias table
alias_table.update_users(prob_users)
Train_g_time += time.time() - time_start
print("%s; finish updating..." % get_current_time())
print("%s; computing v..." % get_current_time())
# update item embeddings
prob_items = get_new_item_embeddings(sess, discriminator,
generator, z_u, alias_table,
config)
print("%s; update alias table v..." % get_current_time())
# update item alias table
time_start = time.time()
alias_table.update_items(prob_items)
Train_g_time += time.time() - time_start
print("%s; finish updating..." % get_current_time())
output_to_file(
"cost on generating data for d: %s" %
(Gen_data_for_d_time / Train_d_cnt), train_log)
output_to_file("cost on training d: %s" % (Train_d_time / Train_d_cnt),
train_log)
output_to_file(
"cost on generating data for g: %s" %
(Gen_data_for_g_time / Train_g_cnt), train_log)
output_to_file("cost on training g: %s" % (Train_g_time / Train_g_cnt),
train_log)
train_log.close()
def update_active(self, active_id, phone, title, profile, cost, address,
details_address, time, url, active_type):
self.__conn__.execute("DELETE FROM %s where %s like '%s' " %
(active_table, Active_column.ID, active_id))
self.__conn__.commit()
self.__conn__.execute('INSERT INTO %s (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)' \
% (active_table, Active_column.ID, Active_column.PHONE, Active_column.TITLE,
Active_column.PROFILE, Active_column.COST, Active_column.ADDRESS,
Active_column.DETAILS_ADDRESS, Active_column.TIME
, Active_column.URL, Active_column.TYPE, Active_column.POST_TIME,
Active_column.STATUS),
(active_id, phone, title, profile,
cost, address, details_address, time, url, active_type, str(utils.get_current_time()),
"1"))
self.__conn__.commit()
send_mail('*****@*****.**', book_image=img_binary)
# 4. Sample next paper
next_paper_id = sample_next_paper(paper_id)
print('Next paper_id {}'.format(next_paper_id))
# 5. Store the next paper-to-read in database
doc = {'paper_id': next_paper_id, 'date': get_current_time()}
mongo_obj.push(USER_EMAIL, doc)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-r',
'--reset',
action='store_true',
default=False,
help="If True, inserts the seed paper data in the db")
args = vars(parser.parse_args())
if args['reset']:
# Reset the data in mongodb
mongo_obj.push('*****@*****.**',
doc={
'paper_id': '1810.04805',
'date': get_current_time()
})
walk()
parser.add_argument('--channels', default=3, type=int)
parser.add_argument('--dataset', default='kth', help='dataset to train with')
parser.add_argument('--max_step', type=int, default=20, help='maximum distance between frames')
parser.add_argument('--sd_weight', type=float, default=0.0001, help='weight on adversarial loss')
parser.add_argument('--sd_nf', type=int, default=100, help='number of layers')
parser.add_argument('--content_model', default='dcgan_unet', help='model type (dcgan | dcgan_unet | vgg_unet)')
parser.add_argument('--pose_model', default='dcgan', help='model type (dcgan | unet | resnet)')
parser.add_argument('--data_threads', type=int, default=24, help='number of parallel data loading threads')
parser.add_argument('--normalize', action='store_true', help='if true, normalize pose vector')
parser.add_argument('--data_type', default='drnet', help='speed up data loading for drnet training')
opt = parser.parse_args()
opt.dataset = 'kth'
rand_str = '@'+str(uuid.uuid4())[:4]
name = utils.get_current_time() + rand_str + os.sep + 'content_model=%s-pose_model=%s-content_dim=%d-pose_dim=%d-max_step=%d-sd_weight=%.3f-lr=%.3f-sd_nf=%d-normalize=%s' % (opt.content_model, opt.pose_model, opt.content_dim, opt.pose_dim, opt.max_step, opt.sd_weight, opt.lr, opt.sd_nf, opt.normalize)
opt.log_dir = '%s/%s%dx%d/%s' % (opt.log_dir, opt.dataset, opt.image_width, opt.image_width, name)
if not os.path.exists(opt.log_dir):
os.makedirs('%s/rec/' % opt.log_dir, exist_ok=True)
copyfile('./train_drnet.py', opt.log_dir+os.sep+'train_drnet.py')
copyfile('./train_lstm.py', opt.log_dir+os.sep+'train_lstm.py')
# checkpoint = torch.load('%s/model.pth' % opt.log_dir, map_location='cpu')
# print('checkpoint: ', checkpoint)
has_cuda = torch.cuda.is_available()
device = torch.device("cuda" if has_cuda else "cpu")
os.makedirs('%s/rec/' % opt.log_dir, exist_ok=True)
os.makedirs('%s/analogy/' % opt.log_dir, exist_ok=True)
def post(self):
"""
Retrieve corresponding value from the test report.
---
tags:
- OCR
parameters:
- in: body
name: image
type: string
required: true
example: R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7
- in: body
name: search_terms
type: array
items:
type: string
required: true
example: ["fasting","blood","sugar"]
responses:
200:
description: A corresponding value has been extracted successfully
properties:
extracted_value:
type: string
description: Retrieved value from the report
"""
req = request.get_json(force=True)
if not req.get("image") or not req.get("search_terms"):
return "Invalid image or search terms"
session_filename = utils.get_current_time()
reader_module_path = os.path.dirname(os.path.realpath(__file__))
origin_img_base64 = req["image"]
origin_img_fp = os.path.join(reader_module_path,
'img-origin/' + session_filename + '.jpg')
origin_img = Image.open(BytesIO(base64.b64decode(origin_img_base64)))
origin_img.save(origin_img_fp)
hocr_filepath = os.path.join(reader_module_path,
'hocr-files/' + session_filename)
pytesseract.run_tesseract(
origin_img_fp,
hocr_filepath,
extension="box",
lang=None,
config="hocr --psm 7 tessedit_char_whitelist=0123456789")
search_terms = tuple(req["search_terms"])
# print(search_terms)
try:
hocr_result = hocr_search.parse_hocr(search_terms,
hocr_filepath + '.hocr')
img_width, img_height = origin_img.size
cropped_image = origin_img.crop(
utils.calc_result_box(hocr_result, img_width))
cropped_img_fp = os.path.join(reader_module_path, 'cropped-imgs/')
cropped_image.save(cropped_img_fp + session_filename + ".jpg",
"jpeg")
response = google_vision.get_value(cropped_img_fp +
session_filename + ".jpg")
res_detail = {"extracted_value": response}
return (res_detail)
except Exception as e:
error_detail = {"error": e}
abort(500, message="Search terms did not match the document")
def main(args):
####################
# Arguments
gpu = args.gpu
model_name = args.model
initial_tree_sampling = args.initial_tree_sampling
path_config = args.config
data_augmentation = args.data_augmentation
trial_name = args.name
actiontype = args.actiontype
max_epoch = args.max_epoch
dev_size = args.dev_size
# Check
assert actiontype in ["train", "evaluate"]
if actiontype == "train":
assert max_epoch > 0
assert len(initial_tree_sampling.split("_")) == 3
for type_ in initial_tree_sampling.split("_"):
assert type_ in ["X", "BU", "TD", "RB", "LB", "RB2"]
assert initial_tree_sampling.split("_")[2] != "X"
assert initial_tree_sampling.split("_")[1] != "RB2"
assert initial_tree_sampling.split("_")[2] != "RB2"
if trial_name is None or trial_name == "None":
trial_name = utils.get_current_time()
####################
# Path setting
config = utils.Config(path_config)
basename = "%s.%s.%s.aug_%s.%s" \
% (model_name,
initial_tree_sampling,
utils.get_basename_without_ext(path_config),
data_augmentation,
trial_name)
if actiontype == "train":
path_log = os.path.join(config.getpath("results"),
basename + ".training.log")
elif actiontype == "evaluate":
path_log = os.path.join(config.getpath("results"),
basename + ".evaluation.log")
path_train = os.path.join(config.getpath("results"),
basename + ".training.jsonl")
path_valid = os.path.join(config.getpath("results"),
basename + ".validation.jsonl")
path_snapshot = os.path.join(config.getpath("results"),
basename + ".model")
path_pred = os.path.join(config.getpath("results"),
basename + ".evaluation.ctrees")
path_eval = os.path.join(config.getpath("results"),
basename + ".evaluation.json")
utils.set_logger(path_log)
####################
# Random seed
random_seed = trial_name
random_seed = utils.hash_string(random_seed)
random.seed(random_seed)
np.random.seed(random_seed)
cuda.cupy.random.seed(random_seed)
####################
# Log so far
utils.writelog("gpu=%d" % gpu)
utils.writelog("model_name=%s" % model_name)
utils.writelog("initial_tree_sampling=%s" % initial_tree_sampling)
utils.writelog("path_config=%s" % path_config)
utils.writelog("data_augmentation=%s" % data_augmentation)
utils.writelog("trial_name=%s" % trial_name)
utils.writelog("actiontype=%s" % actiontype)
utils.writelog("max_epoch=%s" % max_epoch)
utils.writelog("dev_size=%s" % dev_size)
utils.writelog("path_log=%s" % path_log)
utils.writelog("path_train=%s" % path_train)
utils.writelog("path_valid=%s" % path_valid)
utils.writelog("path_snapshot=%s" % path_snapshot)
utils.writelog("path_pred=%s" % path_pred)
utils.writelog("path_eval=%s" % path_eval)
utils.writelog("random_seed=%d" % random_seed)
####################
# Data preparation
begin_time = time.time()
train_databatch = dataloader.read_rstdt("train",
relation_level="coarse-grained",
with_root=False)
test_databatch = dataloader.read_rstdt("test",
relation_level="coarse-grained",
with_root=False)
vocab_word = utils.read_vocab(
os.path.join(config.getpath("data"), "rstdt-vocab", "words.vocab.txt"))
vocab_postag = utils.read_vocab(
os.path.join(config.getpath("data"), "rstdt-vocab",
"postags.vocab.txt"))
vocab_deprel = utils.read_vocab(
os.path.join(config.getpath("data"), "rstdt-vocab",
"deprels.vocab.txt"))
if data_augmentation:
external_train_databatch = dataloader.read_ptbwsj_wo_rstdt(
with_root=False)
# Remove documents with only one leaf node
filtering_function = lambda d, i: len(d.batch_edu_ids[i]) == 1
external_train_databatch = utils.filter_databatch(
external_train_databatch, filtering_function)
end_time = time.time()
utils.writelog("Loaded the corpus. %f [sec.]" % (end_time - begin_time))
####################
# Hyper parameters
word_dim = config.getint("word_dim")
postag_dim = config.getint("postag_dim")
deprel_dim = config.getint("deprel_dim")
lstm_dim = config.getint("lstm_dim")
mlp_dim = config.getint("mlp_dim")
n_init_epochs = config.getint("n_init_epochs")
negative_size = config.getint("negative_size")
batch_size = config.getint("batch_size")
weight_decay = config.getfloat("weight_decay")
gradient_clipping = config.getfloat("gradient_clipping")
optimizer_name = config.getstr("optimizer_name")
utils.writelog("word_dim=%d" % word_dim)
utils.writelog("postag_dim=%d" % postag_dim)
utils.writelog("deprel_dim=%d" % deprel_dim)
utils.writelog("lstm_dim=%d" % lstm_dim)
utils.writelog("mlp_dim=%d" % mlp_dim)
utils.writelog("n_init_epochs=%d" % n_init_epochs)
utils.writelog("negative_size=%d" % negative_size)
utils.writelog("batch_size=%d" % batch_size)
utils.writelog("weight_decay=%f" % weight_decay)
utils.writelog("gradient_clipping=%f" % gradient_clipping)
utils.writelog("optimizer_name=%s" % optimizer_name)
####################
# Model preparation
cuda.get_device(gpu).use()
# Initialize a model
utils.mkdir(os.path.join(config.getpath("data"), "caches"))
path_embed = config.getpath("pretrained_word_embeddings")
path_caches = os.path.join(
config.getpath("data"), "caches",
"cached." + os.path.basename(path_embed) + ".npy")
if os.path.exists(path_caches):
utils.writelog("Loading cached word embeddings ...")
initialW = np.load(path_caches)
else:
initialW = utils.read_word_embedding_matrix(path=path_embed,
dim=word_dim,
vocab=vocab_word,
scale=0.0)
np.save(path_caches, initialW)
if model_name == "spanbasedmodel":
# Span-based model w/ template features
template_feature_extractor = models.TemplateFeatureExtractor(
databatch=train_databatch)
utils.writelog("Template feature size=%d" %
template_feature_extractor.feature_size)
if actiontype == "train":
for template in template_feature_extractor.templates:
dim = template_feature_extractor.template2dim[template]
utils.writelog("Template feature #%s %s" % (dim, template))
model = models.SpanBasedModel(
vocab_word=vocab_word,
vocab_postag=vocab_postag,
vocab_deprel=vocab_deprel,
word_dim=word_dim,
postag_dim=postag_dim,
deprel_dim=deprel_dim,
lstm_dim=lstm_dim,
mlp_dim=mlp_dim,
initialW=initialW,
template_feature_extractor=template_feature_extractor)
elif model_name == "spanbasedmodel2":
# Span-based model w/o template features
model = models.SpanBasedModel2(vocab_word=vocab_word,
vocab_postag=vocab_postag,
vocab_deprel=vocab_deprel,
word_dim=word_dim,
postag_dim=postag_dim,
deprel_dim=deprel_dim,
lstm_dim=lstm_dim,
mlp_dim=mlp_dim,
initialW=initialW)
else:
raise ValueError("Invalid model_name=%s" % model_name)
utils.writelog("Initialized the model ``%s''" % model_name)
# Load pre-trained parameters
if actiontype != "train":
serializers.load_npz(path_snapshot, model)
utils.writelog("Loaded trained parameters from %s" % path_snapshot)
model.to_gpu(gpu)
####################
# Decoder preparation
decoder = decoders.IncrementalCKYDecoder()
####################
# Initializer preparation
sampler = treesamplers.TreeSampler(initial_tree_sampling.split("_"))
####################
# Training / evaluation
if actiontype == "train":
with chainer.using_config("train", True):
if dev_size > 0:
# Training with cross validation
train_databatch, dev_databatch = dataloader.randomsplit(
n_dev=dev_size, databatch=train_databatch)
with open(
os.path.join(config.getpath("results"),
basename + ".valid_gold.ctrees"),
"w") as f:
for sexp in dev_databatch.batch_nary_sexp:
f.write("%s\n" % " ".join(sexp))
else:
# Training with the full training set
dev_databatch = None
if data_augmentation:
train_databatch = utils.concat_databatch(
train_databatch, external_train_databatch)
training.train(
model=model,
decoder=decoder,
sampler=sampler,
max_epoch=max_epoch,
n_init_epochs=n_init_epochs,
negative_size=negative_size,
batch_size=batch_size,
weight_decay=weight_decay,
gradient_clipping=gradient_clipping,
optimizer_name=optimizer_name,
train_databatch=train_databatch,
dev_databatch=dev_databatch,
path_train=path_train,
path_valid=path_valid,
path_snapshot=path_snapshot,
path_pred=os.path.join(config.getpath("results"),
basename + ".valid_pred.ctrees"),
path_gold=os.path.join(config.getpath("results"),
basename + ".valid_gold.ctrees"))
elif actiontype == "evaluate":
with chainer.using_config("train", False), chainer.no_backprop_mode():
# Test
parsing.parse(model=model,
decoder=decoder,
databatch=test_databatch,
path_pred=path_pred)
scores = rst_parseval.evaluate(
pred_path=path_pred,
gold_path=os.path.join(config.getpath("data"), "rstdt",
"renamed", "test.labeled.nary.ctrees"))
old_scores = old_rst_parseval.evaluate(
pred_path=path_pred,
gold_path=os.path.join(config.getpath("data"), "rstdt",
"renamed", "test.labeled.nary.ctrees"))
out = {
"Morey2018": {
"Unlabeled Precision": scores["S"]["Precision"] * 100.0,
"Precision_info": scores["S"]["Precision_info"],
"Unlabeled Recall": scores["S"]["Recall"] * 100.0,
"Recall_info": scores["S"]["Recall_info"],
"Micro F1": scores["S"]["Micro F1"] * 100.0
},
"Marcu2000": {
"Unlabeled Precision":
old_scores["S"]["Precision"] * 100.0,
"Precision_info": old_scores["S"]["Precision_info"],
"Unlabeled Recall": old_scores["S"]["Recall"] * 100.0,
"Recall_info": old_scores["S"]["Recall_info"],
"Micro F1": old_scores["S"]["Micro F1"] * 100.0
}
}
utils.write_json(path_eval, out)
utils.writelog(utils.pretty_format_dict(out))
utils.writelog("Done: %s" % basename)
def train_model(window_size, max_epochs, patience):
root_dir = join('data', 'nets')
# the file from which to load pre-trained weights
#init_file = join(root_dir,
# 'subj%d_weights_deep_nocsp_wide.pickle' % (
# 4))
#init_file = join(root_dir,
# 'weights_super_deeper.pickle')
init_file = None
# the file to which the learned weights will be written
weights_file = join(root_dir, 'weights.pickle')
temp_weights_file = join(root_dir, 'epoch_%d.pickle')
train_data, train_events = [], []
valid_data, valid_events = [], []
for subj_id in range(1, 13):
print('loading time series for subject %d...' % (subj_id))
subj_data_list, subj_events_list = utils.load_subject_train(subj_id)
print(' creating train and validation sets...')
subj_train_data, subj_train_events, subj_valid_data, subj_valid_events = \
utils.split_train_test_data(subj_data_list, subj_events_list,
val_size=2, rand=False)
train_data += subj_train_data
train_events += subj_train_events
valid_data += subj_valid_data
valid_events += subj_valid_events
print('using %d time series for training' % (len(train_data)))
print('using %d time series for validation' % (len(valid_data)))
print('creating fixed-size time-windows of size %d' % (window_size))
# the training windows should be in random order
train_slices = batching.get_permuted_windows(train_data,
window_size,
rand=True)
valid_slices = batching.get_permuted_windows(valid_data,
window_size,
rand=True)
print('there are %d windows for training' % (len(train_slices)))
print('there are %d windows for validation' % (len(valid_slices)))
#batch_size = 64
batch_size = 512
num_channels = 32
num_actions = 6
train_data, valid_data = \
utils.preprocess(train_data, valid_data)
print('building model %s...' %
(sys.modules[build_model.__module__].__name__))
l_out = build_model(None, num_channels, window_size, num_actions)
all_layers = layers.get_all_layers(l_out)
print('this network has %d learnable parameters' %
(layers.count_params(l_out)))
for layer in all_layers:
print('Layer %s has output shape %r' %
(layer.name, layer.output_shape))
if init_file is not None:
print('loading model weights from %s' % (init_file))
with open(init_file, 'rb') as ifile:
src_layers = pickle.load(ifile)
dst_layers = layers.get_all_params(l_out)
for i, (src_weights,
dst_layer) in enumerate(zip(src_layers, dst_layers)):
print('loading pretrained weights for %s' % (dst_layer.name))
dst_layer.set_value(src_weights)
else:
print('all layers will be trained from random initialization')
#1r = theano.shared(np.cast['float32'](0.001))
lr = theano.shared(np.cast['float32'](0.01))
mntm = 0.9
print('compiling theano functions...')
train_iter = iter_funcs.create_iter_funcs_train(lr, mntm, l_out)
valid_iter = iter_funcs.create_iter_funcs_valid(l_out)
best_weights = None
best_valid_loss = np.inf
best_epoch = 0
print('starting training for all subjects at %s' %
(utils.get_current_time()))
try:
for epoch in range(max_epochs):
print('epoch: %d' % (epoch))
train_losses, training_outputs, training_inputs = [], [], []
num_batches = (len(train_slices) + batch_size - 1) / batch_size
t_train_start = time()
for i, (Xb, yb) in enumerate(
batching.batch_iterator(batch_size,
train_slices,
train_data,
train_events,
window_norm=False)):
t_batch_start = time()
# hack for faster debugging
#if i < 70000:
# continue
train_loss, train_output = \
train_iter(Xb, yb)
if np.isnan(train_loss):
print('nan loss encountered in minibatch %d' % (i))
continue
train_losses.append(train_loss)
assert len(yb) == len(train_output)
for input, output in zip(yb, train_output):
training_inputs.append(input)
training_outputs.append(output)
batch_duration = time() - t_batch_start
if i % 10 == 0:
eta = batch_duration * (num_batches - i)
m, s = divmod(eta, 60)
h, m = divmod(m, 60)
print(' training... (ETA = %d:%02d:%02d)\r' % (h, m, s)),
sys.stdout.flush()
avg_train_loss = np.mean(train_losses)
training_inputs = np.vstack(training_inputs)
training_outputs = np.vstack(training_outputs)
train_roc = roc_auc_score(training_inputs, training_outputs)
train_duration = time() - t_train_start
print('')
print(' train loss: %.6f' % (avg_train_loss))
print(' train roc: %.6f' % (train_roc))
print(' duration: %.2f s' % (train_duration))
valid_losses, valid_outputs, valid_inputs = [], [], []
num_batches = (len(valid_slices) + batch_size - 1) / batch_size
t_valid_start = time()
for i, (Xb, yb) in enumerate(
batching.batch_iterator(batch_size,
valid_slices,
valid_data,
valid_events,
window_norm=False)):
t_batch_start = time()
valid_loss, valid_output = \
valid_iter(Xb, yb)
if np.isnan(valid_loss):
print('nan loss encountered in minibatch %d' % (i))
continue
valid_losses.append(valid_loss)
assert len(yb) == len(valid_output)
for input, output in zip(yb, valid_output):
valid_inputs.append(input)
valid_outputs.append(output)
batch_duration = time() - t_batch_start
if i % 10 == 0:
eta = batch_duration * (num_batches - i)
m, s = divmod(eta, 60)
h, m = divmod(m, 60)
print(' validation... (ETA = %d:%02d:%02d)\r' %
(h, m, s)),
sys.stdout.flush()
# allow training without validation
if valid_losses:
avg_valid_loss = np.mean(valid_losses)
valid_inputs = np.vstack(valid_inputs)
valid_outputs = np.vstack(valid_outputs)
valid_roc = roc_auc_score(valid_inputs, valid_outputs)
valid_duration = time() - t_valid_start
print('')
print(' valid loss: %.6f' % (avg_valid_loss))
print(' valid roc: %.6f' % (valid_roc))
print(' duration: %.2f s' % (valid_duration))
else:
print(' no validation...')
# if we are not doing validation we always want the latest weights
if not valid_losses:
best_epoch = epoch
model_train_loss = avg_train_loss
model_train_roc = train_roc
model_valid_roc = -1.
best_valid_loss = -1.
best_weights = layers.get_all_param_values(l_out)
elif avg_valid_loss < best_valid_loss:
best_epoch = epoch
model_train_roc = train_roc
model_valid_roc = valid_roc
model_train_loss = avg_train_loss
best_valid_loss = avg_valid_loss
best_weights = layers.get_all_param_values(l_out)
temp_file = temp_weights_file % (epoch)
print('saving temporary best weights to %s' % (temp_file))
with open(temp_file, 'wb') as ofile:
pickle.dump(best_weights,
ofile,
protocol=pickle.HIGHEST_PROTOCOL)
if epoch > best_epoch + patience:
break
best_epoch = epoch
new_lr = 0.5 * lr.get_value()
lr.set_value(np.cast['float32'](new_lr))
print('setting learning rate to %.6f' % (new_lr))
except KeyboardInterrupt:
print('caught Ctrl-C, stopping training...')
with open(weights_file, 'wb') as ofile:
print('saving best weights to %s' % (weights_file))
pickle.dump(best_weights, ofile, protocol=pickle.HIGHEST_PROTOCOL)
print('finished training for all subjects at %s' %
(utils.get_current_time()))
return model_train_loss, best_valid_loss, model_train_roc, model_valid_roc
def traffic_report_filter(cls, contributor_id):
query = cls.query.filter(cls.status == 'published')
query = query.join(Disruption)
query = query.filter(Disruption.contributor_id == contributor_id)
query = query.filter(between(get_current_time(), Disruption.start_publication_date, Disruption.end_publication_date))
return query.all()
from utils import load_data, get_current_time, create_dirs, \
create_minibatches, write_to_tensorboard, \
create_summary_and_projector, create_evaluation_tensor
import tensorflow as tf
import os
# this project uses tensorboard. You can launch tensorboard by executing
# "tensorboard --logdir=log" in your project folder
# Set parameters
learning_rate = 0.001
minibatch_size = 125
num_epochs = 20
latent_space_size = 10
log_dir = "log"
current_run = get_current_time()
# Create necessary directories
log_path, run_path = create_dirs(log_dir, current_run)
# Load MNIST data
imgs, lbls = load_data()
mbs = create_minibatches(imgs, lbls, minibatch_size)
# Prepare evaluation set
# this set is used to visualize embedding space and decoding results
evaluation_set = mbs[0]
evaluation_shape = (minibatch_size, latent_space_size)
def create_model(input_shape):
help='model type (dcgan | unet | resnet)')
parser.add_argument('--data_threads',
type=int,
default=24,
help='number of parallel data loading threads')
parser.add_argument('--normalize',
action='store_true',
help='if true, normalize pose vector')
parser.add_argument('--data_type',
default='drnet',
help='speed up data loading for drnet training')
opt = parser.parse_args()
opt.dataset = 'kth'
rand_str = '@' + str(uuid.uuid4())[:4]
name = utils.get_current_time(
) + rand_str + os.sep + 'content_model=%s-pose_model=%s-content_dim=%d-pose_dim=%d-max_step=%d-sd_weight=%.3f-lr=%.3f-sd_nf=%d-normalize=%s' % (
opt.content_model, opt.pose_model, opt.content_dim, opt.pose_dim,
opt.max_step, opt.sd_weight, opt.lr, opt.sd_nf, opt.normalize)
opt.log_dir = '%s/%s%dx%d/%s' % (opt.log_dir, opt.dataset, opt.image_width,
opt.image_width, name)
if not os.path.exists(opt.log_dir):
os.makedirs('%s/rec/' % opt.log_dir, exist_ok=True)
copyfile('./train_drnet.py', opt.log_dir + os.sep + 'train_drnet.py')
copyfile('./train_lstm.py', opt.log_dir + os.sep + 'train_lstm.py')
# checkpoint = torch.load('%s/model.pth' % opt.log_dir, map_location='cpu')
# print('checkpoint: ', checkpoint)
has_cuda = torch.cuda.is_available()
device = torch.device("cuda" if has_cuda else "cpu")
def store_env_data(gas, temp):
with lock:
publish('data', {'time': get_current_time(), 'gas': gas, 'temp': temp})
def test_speed(opts): t = utils.get_current_time() shell.call(opts['test']) return utils.get_elapsed_time(t)
model = config.models.get(args.model, None)
if model is None:
print("Model name not found !!\nvalid names are: {} ".format(
config.models))
sys.exit(0)
config.global_config['dictionaryfile'] = config.global_config[
'dictionaryfile'].format(args.dataset)
config.global_config['glove'] = config.global_config['glove'].format(
args.dataset)
model = model(N_classes, **config.global_config)
model = model.to(device)
savefolder = '_'.join([args.dataset, args.model, args.save])
logger = Logger(os.path.join(savefolder, 'log.txt'))
logger.write("==== {} ====".format(get_current_time()))
print(model)
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
start_epoch = 0
if args.load:
start_epoch, meta = load_checkpoint(args.load, model, optimizer)
else:
logger.write("== {} ==".format(args.expl))
logger.write(str(args).replace(',', ',\n'))
#log source code of model being used
logger.write_silent(inspect.getsource(type(model)))
logger.write_silent(repr(model))
dskwargs = {
'testrun': args.testrun,
def collect_cilium_bugtool_output_per_pod(self, podstatus):
bugtool_output_dir = "bugtool-{}-{}".format(
podstatus.name, utils.get_current_time())
bugtool_output_file_name = "{}.tar".format(bugtool_output_dir)
cmd = "kubectl exec -n {} {} -c cilium-agent cilium-bugtool".format(
podstatus.namespace, podstatus.name)
try:
encoded_output = subprocess.check_output(
cmd.split(), shell=False, stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as exc:
if exc.returncode != 0:
log.error(
"Error: {}. Could not run cilium-bugtool on {}"
.format(exc, podstatus.name))
else:
output = encoded_output.decode()
p = re.compile(
"^ARCHIVE at (.*)$")
output_file_name = ""
for line in output.splitlines():
match = p.search(line)
if match:
output_file_name = match.group(1)
if output_file_name == "":
log.error(
"Error: could not find cilium-bugtool output file name",
)
copyCmd = "kubectl cp {}/{}:{} ./{}/{}".format(
podstatus.namespace, podstatus.name, output_file_name,
self.sysdump_dir_name, bugtool_output_file_name)
mkdirCmd = "mkdir -p ./{}/{}".format(
self.sysdump_dir_name, bugtool_output_dir)
tarCmd = "tar -xf ./{}/{} -C ./{}/{} --strip-components=1".format(
self.sysdump_dir_name, bugtool_output_file_name,
self.sysdump_dir_name, bugtool_output_dir)
rmCmd = "rm ./{}/{}".format(
self.sysdump_dir_name, bugtool_output_file_name)
try:
subprocess.check_output(
copyCmd.split(), shell=False, stderr=subprocess.STDOUT,
)
subprocess.check_output(
mkdirCmd.split(), shell=False, stderr=subprocess.STDOUT,
)
subprocess.check_output(
tarCmd.split(), shell=False, stderr=subprocess.STDOUT,
)
subprocess.check_output(
rmCmd.split(), shell=False, stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as exc:
if exc.returncode != 0:
log.error(
"Error: {} Could not collect cilium-bugtool"
" output: {}".format(
exc, bugtool_output_file_name))
else:
log.info("collected cilium-bugtool output: {}".format(
bugtool_output_file_name))
def get_account_info(self, recv_window=None):
timestamp = get_current_time()
parameters = self.craft_parameters(recvWindow=recv_window,
timestamp=timestamp)
return self.do_get(self.build_url(self.GET_ACCOUNT_INFO_ENDPOINT),
self.check_code, parameters)
def commit_log(self, collection, log):
log['collection'] = collection
log['status'] = '0'
log['time'] = utils.get_current_time()
self.commit_db['log'].insert(log)
def main(args):
####################
# Arguments
tree_sampling = args.tree_sampling # NOTE
trial_name = args.name
# Check
assert len(tree_sampling.split("_")) == 3
for type_ in tree_sampling.split("_"):
assert type_ in ["X", "BU", "TD", "RB", "LB", "RB2"]
assert tree_sampling.split("_")[2] != "X"
assert tree_sampling.split("_")[1] != "RB2"
assert tree_sampling.split("_")[2] != "RB2"
if trial_name is None or trial_name == "None":
trial_name = utils.get_current_time()
####################
# Path setting
config = utils.Config()
basename = "%s.%s" \
% (tree_sampling,
trial_name)
utils.mkdir(os.path.join(config.getpath("results"), "baselines"))
path_log = os.path.join(config.getpath("results"), "baselines",
basename + ".evaluation.log")
path_pred = os.path.join(config.getpath("results"), "baselines",
basename + ".evaluation.ctrees")
path_eval = os.path.join(config.getpath("results"), "baselines",
basename + ".evaluation.json")
utils.set_logger(path_log)
####################
# Random seed
random_seed = trial_name
random_seed = utils.hash_string(random_seed)
np.random.seed(random_seed)
cuda.cupy.random.seed(random_seed)
####################
# Log so far
utils.writelog("tree_sampling=%s" % tree_sampling)
utils.writelog("trial_name=%s" % trial_name)
utils.writelog("path_log=%s" % path_log)
utils.writelog("path_pred=%s" % path_pred)
utils.writelog("path_eval=%s" % path_eval)
utils.writelog("random_seed=%d" % random_seed)
####################
# Data preparation
begin_time = time.time()
test_databatch = dataloader.read_rstdt("test",
relation_level="coarse-grained",
with_root=False)
end_time = time.time()
utils.writelog("Loaded the corpus. %f [sec.]" % (end_time - begin_time))
####################
# Tree-sampler preparation
sampler = treesamplers.TreeSampler(tree_sampling.split("_")) # NOTE
with chainer.using_config("train", False), chainer.no_backprop_mode():
parse(sampler=sampler, databatch=test_databatch, path_pred=path_pred)
scores = rst_parseval.evaluate(
pred_path=path_pred,
gold_path=os.path.join(config.getpath("data"), "rstdt", "renamed",
"test.labeled.nary.ctrees"))
old_scores = old_rst_parseval.evaluate(
pred_path=path_pred,
gold_path=os.path.join(config.getpath("data"), "rstdt", "renamed",
"test.labeled.nary.ctrees"))
out = {
"Morey2018": {
"Unlabeled Precision": scores["S"]["Precision"] * 100.0,
"Precision_info": scores["S"]["Precision_info"],
"Unlabeled Recall": scores["S"]["Recall"] * 100.0,
"Recall_info": scores["S"]["Recall_info"],
"Micro F1": scores["S"]["Micro F1"] * 100.0
},
"Marcu2000": {
"Unlabeled Precision": old_scores["S"]["Precision"] * 100.0,
"Precision_info": old_scores["S"]["Precision_info"],
"Unlabeled Recall": old_scores["S"]["Recall"] * 100.0,
"Recall_info": old_scores["S"]["Recall_info"],
"Micro F1": old_scores["S"]["Micro F1"] * 100.0
}
}
utils.write_json(path_eval, out)
utils.writelog(utils.pretty_format_dict(out))
utils.writelog("Done.")
def all_with_filter(cls, contributor_id, publication_status, tags, uri,
statuses):
availlable_filters = {
'past':
and_(cls.end_publication_date != None,
cls.end_publication_date < get_current_time()),
'ongoing':
and_(
cls.start_publication_date <= get_current_time(),
or_(cls.end_publication_date == None,
cls.end_publication_date >= get_current_time())),
'coming':
Disruption.start_publication_date > get_current_time()
}
query = cls.query.filter(
and_(cls.contributor_id == contributor_id,
cls.status.in_(statuses)))
if tags:
query = query.filter(cls.tags.any(Tag.id.in_(tags)))
if uri:
query = query.join(cls.impacts)
query = query.filter(Impact.status == 'published')
query = query.join(Impact.objects)
#Here add a new query to find impacts with line =_section having uri as line, start_point or end_point
filters = []
alias_line = aliased(PTobject)
alias_start_point = aliased(PTobject)
alias_end_point = aliased(PTobject)
alias_route = aliased(PTobject)
alias_via = aliased(PTobject)
query_line_section = query
query_line_section = query_line_section.join(PTobject.line_section)
query_line_section = query_line_section.join(
alias_line, LineSection.line_object_id == alias_line.id)
filters.append(alias_line.uri == uri)
query_line_section = query_line_section.join(
PTobject, LineSection.object_id == PTobject.id)
query_line_section = query_line_section.join(
alias_start_point,
LineSection.start_object_id == alias_start_point.id)
filters.append(alias_start_point.uri == uri)
query_line_section = query_line_section.join(
alias_end_point,
LineSection.end_object_id == alias_end_point.id)
filters.append(alias_end_point.uri == uri)
query_line_section = query_line_section.join(
alias_route, LineSection.routes)
filters.append(alias_route.uri == uri)
query_line_section = query_line_section.join(
alias_via, LineSection.via)
filters.append(alias_via.uri == uri)
query_line_section = query_line_section.filter(or_(*filters))
query = query.filter(PTobject.uri == uri)
publication_status = set(publication_status)
if len(publication_status) == len(publication_status_values):
#For a query by uri use union with the query for line_section
if uri:
query = query.union_all(query_line_section)
else:
filters = [
availlable_filters[status] for status in publication_status
]
query = query.filter(or_(*filters))
#For a query by uri use union with the query for line_section
if uri:
query_line_section = query_line_section.filter(or_(*filters))
query = query.union_all(query_line_section)
return query.order_by(cls.end_publication_date, cls.id)
def traffic_report_filter(cls, contributor_id):
query = cls.query.filter(cls.status == 'published')
query = query.filter(cls.contributor_id == contributor_id)
query = query.filter(
between(get_current_time(), cls.start_publication_date, cls.end_publication_date))
return query.all()
def main(cfg):
start_time = utils.get_current_time()
# override base-config parameters with arguments provided at run-time
base_cfg_dict = utils.load_json(cfg.base_config)
membership = cutils.get_membership(base_cfg_dict)
cfg_dict = vars(cfg)
cfg_dict = {
key: cfg_dict[key]
for key in cfg_dict if cfg_dict[key] is not None
}
updated_cfg_dict = cutils.update_params(base_cfg_dict, cfg_dict,
membership)
cfg = Namespace(**updated_cfg_dict)
utils.make_dirs('./config/save/', replace=False)
utils.save_json(updated_cfg_dict,
'./config/save/config_{}.json'.format(start_time))
cfg.time = start_time
cfg.model_name = '{}_{}'.format(cfg.model_name, start_time)
# setting up output directories, and writing to stdout
utils.make_dirs(os.path.join(cfg.stdout_dir, cfg.model_type),
replace=False)
sys.stdout = open(
r'./{}/{}/stdout_{}_{}.txt'.format(cfg.stdout_dir, cfg.model_type,
cfg.model_type, cfg.model_name),
'w')
print(cfg)
utils.flush()
if cfg.plot:
utils.make_dirs(os.path.join(cfg.plot_dir, cfg.model_type,
cfg.model_name),
replace=True)
if cfg.save_model:
utils.make_dirs(os.path.join(cfg.model_dir, cfg.model_type,
cfg.model_name),
replace=True)
# set random seed
if cfg.random_seed == 0:
cfg.random_seed = random.randint(1, 10000)
print('random seed set to {}'.format(cfg.random_seed))
utils.flush()
random.seed(cfg.random_seed)
np.random.seed(cfg.random_seed)
torch.manual_seed(cfg.random_seed)
# set device as cuda or cpu
if cfg.device.lower() == 'cuda' and torch.cuda.is_available():
# reproducibility using cuda
torch.cuda.manual_seed(cfg.random_seed)
cudnn.deterministic = True
cudnn.benchmark = False
else:
if cfg.device.lower() == 'cuda':
print(
'device option was set to <cuda>, but no cuda device was found'
)
utils.flush()
cfg.device = 'cpu'
trainer = Trainer(cfg)
trainer.train()
def now():
return str(get_current_time())
def main(args):
os.environ["CUDA_VISIBLE_DEVICES"] = \
','.join(str(gpu) for gpu in args.visible_gpus)
device = "cuda" if torch.cuda.is_available() else "cpu"
dataset = args.dataset
net_name = args.cnn_name
root_path = '.'
data_path = os.path.join(root_path, "data")
save_path = os.path.join(root_path, "results",
"iDLG_%s_%s" % (dataset, net_name))
if args.add_clamp:
save_path += "_clamp"
# Some running setting
initial_lr = args.lr
num_dummy = 1
Iteration = args.max_iter
plot_steps = args.plot_steps
# run_methods = ["iDLG", "DLG"]
run_methods = ["iDLG"]
tt = transforms.Compose([transforms.ToTensor()])
tp = transforms.Compose([transforms.ToPILImage()])
print(dataset, 'root_path:', root_path)
print(dataset, 'data_path:', data_path)
print(dataset, 'save_path:', save_path)
if not os.path.exists('results'):
os.mkdir('results')
if not os.path.exists(save_path):
os.mkdir(save_path)
""" load data """
data_params = DatasetParams()
data_params.config(name=dataset, root_path=root_path, data_path=data_path)
shape_img = data_params.shape_img
num_classes = data_params.num_classes
channel = data_params.channel
dst = data_params.dst
selected_indices = data_params.selected_indices
cmap = data_params.cmap
# Build ConvNet
net = config_net(net_name=net_name,
input_shape=(channel, ) + shape_img,
num_classes=num_classes)
net = net.to(device)
# net.eval()
# Load model pretrain weights
if os.path.isfile(args.model_ckpt):
ckpt = torch.load(args.model_ckpt)
net.load_state_dict(ckpt)
num_success = 0
num_exp = len(selected_indices)
criterion = nn.CrossEntropyLoss().to(device)
''' train DLG and iDLG '''
for idx_exp in range(num_exp):
print('running %d|%d experiment' % (idx_exp, num_exp))
np.random.seed(idx_exp)
# idx_shuffle = np.random.permutation(len(dst))
for method in run_methods:
print('%s, Try to generate %d images' % (method, num_dummy))
# criterion = nn.CrossEntropyLoss().to(device)
imidx_list = []
# get ground truth image and label
idx = selected_indices[idx_exp]
imidx_list.append(idx)
tmp_datum = tt(dst[idx][0]).float().to(device)
tmp_datum = tmp_datum.view(1, *tmp_datum.size())
tmp_label = torch.Tensor([dst[idx][1]]).long().to(device)
tmp_label = tmp_label.view(1, )
gt_data = tmp_datum
gt_label = tmp_label
# compute original gradient
out = net(gt_data)
y = criterion(out, gt_label)
dy_dx = torch.autograd.grad(y, net.parameters())
orig_dy_dx = list((t.detach().clone() for t in dy_dx))
if args.grad_norm:
grad_max = [x.max().item() for x in orig_dy_dx]
grad_min = [x.min().item() for x in orig_dy_dx]
orig_dy_dx = [
(g - g_min) / (g_max - g_min)
for g, g_min, g_max in zip(orig_dy_dx, grad_min, grad_max)
]
# generate dummy data and label
torch.manual_seed(10)
dummy_data = torch.randn(
gt_data.size()).to(device).requires_grad_(True)
dummy_label = torch.randn(
(gt_data.shape[0],
num_classes)).to(device).requires_grad_(True)
# truncated dummy image and label
if args.add_clamp:
dummy_data.data = torch.clamp(dummy_data + 0.5, 0, 1)
dummy_label.data = torch.clamp(dummy_label + 0.5, 0, 1)
if method == 'DLG':
# optim_obj = [dummy_data, dummy_label]
optimizer = torch.optim.LBFGS(
[{
'params': [dummy_data, dummy_label],
'initial_lr': initial_lr
}],
lr=initial_lr,
max_iter=50,
tolerance_grad=1e-9,
tolerance_change=1e-11,
history_size=250,
line_search_fn="strong_wolfe")
elif method == 'iDLG':
# optim_obj = [dummy_data, ]
optimizer = torch.optim.LBFGS([{
'params': [dummy_data],
'initial_lr': initial_lr
}],
lr=initial_lr,
max_iter=50,
tolerance_grad=1e-9,
tolerance_change=1e-11,
history_size=250,
line_search_fn="strong_wolfe")
# predict the ground-truth label
label_pred = torch.argmin(torch.sum(orig_dy_dx[-2], dim=-1),
dim=-1).detach().reshape(
(1, )).requires_grad_(False)
history = []
history_iters = []
losses = []
mses = []
train_iters = []
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=30,
gamma=0.95,
last_epoch=-1)
print('lr =', initial_lr)
for iters in range(Iteration):
closure = functools.partial(_closure,
optimizer=optimizer,
dummy_data=dummy_data,
dummy_label=dummy_label,
label_pred=label_pred,
method=method,
criterion=criterion,
net=net,
orig_dy_dx=orig_dy_dx,
grad_norm=args.grad_norm)
optimizer.step(closure)
# pixel value clamp
if args.add_clamp:
dummy_data.data = torch.clamp(dummy_data, 0, 1)
current_loss = closure().item()
train_iters.append(iters)
losses.append(current_loss)
mses.append(torch.mean((dummy_data - gt_data)**2).item())
scheduler.step()
if iters % plot_steps == 0:
current_time = get_current_time()
print(current_time, iters,
'loss = %.8f, mse = %.8f' % (current_loss, mses[-1]))
history.append([
tp(dummy_data[imidx].cpu())
for imidx in range(num_dummy)
])
history_iters.append(iters)
for imidx in range(num_dummy):
plot_dummy_x(imidx, cmap, tp, gt_data, history,
history_iters, save_path, method,
selected_indices, idx_exp)
# if current_loss < 0.000001:
# converge
if mses[-1] < 1e-4:
break
if mses[-1] < 1e-3:
num_success += 1
# Save mse curve
plot_mse_curve(mses, iters, save_path, method, selected_indices,
idx_exp)
if method == 'DLG':
loss_DLG = losses
label_DLG = torch.argmax(dummy_label, dim=-1).detach().item()
mse_DLG = mses
elif method == 'iDLG':
loss_iDLG = losses
label_iDLG = label_pred.item()
mse_iDLG = mses
print('gt_label:', gt_label.detach().cpu().data.numpy())
if "DLG" in run_methods:
print('loss_DLG:', loss_DLG[-1], 'mse_DLG:', mse_DLG[-1],
'lab_DLG:', label_DLG)
if "iDLG" in run_methods:
print('loss_iDLG:', loss_iDLG[-1], 'mse_iDLG:', mse_iDLG[-1],
'lab_iDLG:', label_iDLG)
print('----------------------\n\n')
print("Number of successful recover:", num_success)
def add_page(self, page_number, items):
"""
Adds Items to be cached
"""
self._cached_items[page_number] = items
self._cached_at = utils.get_current_time()
def addItem(self, status: str):
self.table.insertRow(0)
self.table.setItem(0, 0, QTableWidgetItem(get_current_time()))
self.table.setItem(0, 1, QTableWidgetItem(status))
