def init():
part_of_body = 'WRIST'
phase_cat = ['train', 'valid']
case_data = get_study_data('XR_' + part_of_body, 'D:/MURA-v1.1/{0}/{1}/')
dataloaders = get_dataloaders(case_data, batch_size=1)
# train abnormal, normal images
tai = {x: get_count(case_data[x], 'positive') for x in phase_cat}
tni = {x: get_count(case_data[x], 'negative') for x in phase_cat}
Wt1 = {x: np_tensor(tni[x] / (tni[x] + tai[x])) for x in phase_cat}
Wt0 = {x: np_tensor(tai[x] / (tni[x] + tai[x])) for x in phase_cat}
def try_get_from(ch, cont, arg):
'''tries to get one item from inside another'''
if not cont.istype("container"):
mud.message(ch, None, cont, None, True, "to_char",
"$o is not a container.")
elif cont.container_is_closed:
mud.message(ch, None, cont, None, True, "to_char", "$o is closed.")
else:
# find our count and name
num, name = utils.get_count(arg)
# multi or single?
if num == "all":
list = utils.find_all_objs(ch, cont.objs, name)
for obj in list:
do_get(ch, obj, cont)
else:
# obj = find_obj(ch, cont.objs, num, name)
obj = mudobj.find_obj(arg, cont, ch)
if obj != None:
do_get(ch, obj, cont)
else:
mud.message(
ch, None, cont, None, True, "to_char",
"You could not find what you were looking for in $o.")
def collate_voters(congress_num):
import time
t1 = time.time()
votes = congress_votes(congress_num)
print len(votes)
atr = [{'name':'category', 'type':'in', 'value':['passage',
'passage_part']},]
purged = purge(votes, atr)
ids = [ str(p['id']) for p in purged ]
id_str = '|'.join(ids)
count_url = "http://www.govtrack.us/api/v2/vote_voter?vote__in=%s" % (id_str)
count = get_count(count_url)
url = "http://www.govtrack.us/api/v2/vote_voter?limit=%s&vote__in=%s" % (count, id_str)
member_votes = dl(url)['objects']
collated_votes = {}
print len(member_votes)
for vote in member_votes:
vote_id = vote['vote']['id']
key = vote['option']['key']
member = vote['person']['id']
if not collated_votes.has_key(vote_id):
collated_votes[vote_id] = [[],[],[],[]]
key_index = key_mapping[key]
collated_votes[vote_id][key_index].append(member)
print "time= %s" % (time.time()-t1)
return collated_votes
def create_equal_dataset(input_data, output_data, min_num) -> (np.array, np.array):
"""
creates an equally sampled dataset based on the given labels and their count
:param input_data: features
:param output_data: labels
:param min_num: minimum number of samples to contain
"""
filtered_inputs = []
filtered_outputs = []
labels_count = get_count(output_data)
num_samples = dict()
for key in labels_count.keys():
num_samples[key] = 0
_, label_count = min_label_count(labels_count)
label_count = min(min_num, label_count)
for i in range(len(output_data)):
label = output_data[i]
if num_samples[label] < label_count:
filtered_inputs.append(input_data[i])
filtered_outputs.append(label)
num_samples[label] += 1
return np.array(filtered_inputs), np.array(filtered_outputs)
def __init__(self, m_name, m_id, epsilon, delay, intervention_file,
monitor_file, out_file, error_file, status_file,
my_counter_file, global_counter_file, end_date):
self.api = Api(PUB_KEY, PRIV_KEY)
self.m_name = m_name
self.m_id = int(m_id)
self.epsilon = epsilon
self.delay = delay
self.intervention_file = intervention_file
self.monitor_file = monitor_file
self.out_file = out_file
self.error_file = error_file
self.my_counter_file = my_counter_file
self.status_file = status_file
self.global_counter_file = global_counter_file
self.intervention_num = utils.get_count(my_counter_file)
self.end_date = end_date
self.set_status('ok')
self.monitor_num = -1
self.init_queue()
self.control_trade_id = None
self.cancel_trades()
def train_model(dataset, model_type, train_fn, model_file, **kwargs):
num_epochs = kwargs['num_epochs']
num_features = kwargs['num_features']
file_prefix = kwargs['file_prefix']
print("Loading dataset...")
input_train_file = dataset + "/" + file_prefix + "_in"
output_train_file = dataset + "/" + file_prefix + "_out"
inputs = features_from_file(input_train_file, num_features)
inputs = transpose_vector(inputs)
outputs = labels_from_file(output_train_file)
label_count = get_count(outputs)
print(label_count)
print("Finished loading dataset")
print(inputs.shape)
model = model_type(len(label_count))
print("Training model..")
model = train_fn(model_file, inputs, outputs, model, num_epochs=num_epochs)
print("Done training model...")
model.save(model_file + ".model")
def create_lang_dataset(out_data_path, min_samples=0):
if out_data_path[-1] != '/':
out_data_path = out_data_path + '/'
if min_samples <= 0:
min_samples = 2 ** 20
en_input, en_output = get_data("accent", english_dataset_path, file_list[5], out_accent_file)
fr_input, fr_output = get_data("accent", french_dataset_path, file_list[5], out_accent_file)
de_input, de_output = get_data("accent", german_dataset_path, file_list[5], out_accent_file)
en_input = en_input[:min_samples]
fr_input = fr_input[:min_samples]
de_input = de_input[:min_samples]
en_output = ["english" for _ in range(len(en_input))]
fr_output = ["french" for _ in range(len(fr_input))]
de_output = ["german" for _ in range(len(de_input))]
inputs = en_input + fr_input + de_input
outputs = en_output + fr_output + de_output
print(len(inputs))
print(len(outputs))
print(get_count(outputs))
get_features(out_data_path + "lang_", inputs, ['delta', 'delta2', 'sdc'])
write_to_file_labels(out_data_path + "lang_out", outputs)
in_files = ["lang_input" + str(i + 1) for i in range(6)]
concat_files(out_data_path, in_files, "lang_in")
def create_gender_dataset(out_data_path, min_samples=0):
"""
create the files holding the data for the gender prediction model
:param out_data_path: where to save the files
:param min_samples: minimum number of samples
"""
if out_data_path[-1] != '/':
out_data_path = out_data_path + '/'
if min_samples <= 0:
min_samples = 2 ** 20
en_input, en_output = get_data(["gender", "age"], english_dataset_path,
file_list[5], out_gender_file)
inputs, outputs = clean_gender_dataset(en_input, en_output)
inputs, outputs = create_equal_dataset(inputs, outputs, min_samples)
print(len(inputs))
print(len(outputs))
print(get_count(outputs))
get_features(out_data_path + "gender_", inputs, ['delta', 'delta2', 'pitch'])
write_to_file_labels(out_data_path + "gender_out", outputs)
in_files = ["gender_input" + str(i + 1) for i in range(6)]
concat_files(out_data_path, in_files, "gender_in")
def get_points(self, query, count=10, scope='*'):
# type: (str, int, str) -> osisoftpy.structures.TypedList[Point]
"""
:param query:
:param count:
:param scope:
:return:
"""
payload = {'q': query, 'count': count, 'scope': scope}
log.debug(
'Executing Query against PI Web WebAPI Indexed Search with '
'the following parameters: Query: "%s", Count: "%s". Payload: %s',
query, count, payload)
r = self.session.get(self.url + '/search/query', params=payload)
if r.status_code != requests.codes.ok:
r.raise_for_status()
else:
data = r.json()
log.debug('HTTP %s - Instantiating %s PI points', r.status_code,
get_count(data.get('Items', None)))
factory = Factory(Point)
items = list(map(lambda x: create(factory, x),
data.get('Items', None)))
points = TypedList(Point)
for point in items:
points.append(point)
log.debug('PI Point retrieval success! %s PI '
'point(s) were '
'found and instantiated.', get_count(points))
if len(data['Errors']) != 0:
for error in data['Errors']:
try:
log.warning('The PI Web WebAPI returned the '
'following error while instantiating '
'PI points. '
'ErrorCode: {0}, Source: {1}, '
'Message {2}'.format(
error['ErrorCode'], error['Source'],
error['Message']))
except Exception as e:
log.error('Exception encounted while '
'instantiating '
'PI points!', exc_info=True)
return points
def dataset_plot(data_file):
labels = labels_from_file(data_file)
counts = get_count(labels)
counts_x = list(counts.keys())
counts_y = list(counts.values())
plt.bar([i * 2 for i in range(len(counts_x))],
counts_y,
tick_label=counts_x)
plt.show()
def get_data_archive_servers(self):
# type: () -> osisoftpy.structures.TypedList[DataArchive]
"""
:return:
"""
log.debug('Retrieving all PI Data Archive servers from %s', self.url)
r = self.session.get(self.url + '/dataservers')
if r.status_code == requests.codes.ok:
data = r.json()
if len(data['Items']) > 0:
log.debug('HTTP %s - Instantiating OSIsoftPy.DataArchives()',
r.status_code)
factory = Factory(DataArchive)
servers = TypedList(validtypes=DataArchive)
log.debug('Staging %s PI server(s) for instantiation...',
get_count(data['Items']))
for i in data['Items']:
try:
log.debug('Instantiating "%s" as '
'OSIsoftPy.DataArchive...', i['Name'])
server = factory.create(name=i['Name'],
serverversion=i[
'ServerVersion'],
webid=i['WebId'],
isconnected=i['IsConnected'],
id=i['Id'])
servers.append(server)
except OSIsoftPyException as e:
log.error('Unable to retrieve server info for '
'"%s"', i['Name'], exc_info=True)
log.debug('PI Data Archive server retrieval success! %s PI '
'server(s) were '
'found and instantiated.', get_count(servers))
return servers
r.raise_for_status()
def create_age_dataset(out_data_path, min_samples=0):
if out_data_path[-1] != '/':
out_data_path = out_data_path + '/'
if min_samples <= 0:
min_samples = 2 ** 20
en_input, en_output = get_data("age", english_dataset_path, file_list[5], out_age_file)
inputs, outputs = clean_age_dataset(en_input, en_output)
inputs, outputs = create_equal_dataset(inputs, outputs, min_samples)
print(len(inputs))
print(len(outputs))
print(get_count(outputs))
get_features(out_data_path + "age_", inputs, ['delta', 'delta2', 'pitch'])
write_to_file_labels(out_data_path + "age_out", outputs)
in_files = ["age_input" + str(i + 1) for i in range(6)]
concat_files(out_data_path, in_files, "age_in")
def try_get_from(ch, cont, arg):
'''tries to get one item from inside another'''
if not cont.istype("container"):
mud.message(ch, None, cont, None, True, "to_char",
"$o is not a container.")
elif cont.container_is_closed:
mud.message(ch, None, cont, None, True, "to_char", "$o is closed.")
else:
# find our count and name
num, name = utils.get_count(arg)
# multi or single?
if num == "all":
list = utils.find_all_objs(ch, cont.objs, name)
for obj in list:
do_get(ch, obj, cont)
else:
# obj = find_obj(ch, cont.objs, num, name)
obj = mudobj.find_obj(arg, cont, ch)
if obj != None:
do_get(ch, obj, cont)
else:
mud.message(ch, None, cont, None, True, "to_char",
"You could not find what you were looking for in $o.")
from densenet import densenet169
from utils import plot_training, n_p, get_count
from train import train_model, get_metrics
from pipeline import get_study_level_data, get_dataloaders
# #### load study level dict data
study_data = get_study_level_data(study_type='XR_WRIST')
# #### Create dataloaders pipeline
data_cat = ['train', 'valid'] # data categories
dataloaders = get_dataloaders(study_data, batch_size=1)
dataset_sizes = {x: len(study_data[x]) for x in data_cat}
# #### Build model
# tai = total abnormal images, tni = total normal images
tai = {x: get_count(study_data[x], 'positive') for x in data_cat}
tni = {x: get_count(study_data[x], 'negative') for x in data_cat}
Wt1 = {x: n_p(tni[x] / (tni[x] + tai[x])) for x in data_cat}
Wt0 = {x: n_p(tai[x] / (tni[x] + tai[x])) for x in data_cat}
print('tai:', tai)
print('tni:', tni, '\n')
print('Wt0 train:', Wt0['train'])
print('Wt0 valid:', Wt0['valid'])
print('Wt1 train:', Wt1['train'])
print('Wt1 valid:', Wt1['valid'])
class Loss(torch.nn.modules.Module):
def __init__(self, Wt1, Wt0):
super(Loss, self).__init__()
def get_values(self, points, calculationtype=None, starttime='*-1d',
endtime='*', boundary=None, boundarytype=None,
maxcount='1000', desiredunits=None, interval=None,
intervals='24', retrievalmode='Auto', summarytype='Total',
calculationbasis='TimeWeighted', timetype='Auto',
summaryduration=None, sampletype='ExpressionRecordedValues',
sampleinterval=None, time=None, filterexpression=None,
includefilteredvalues=False, sortorder='Ascending',
append=False, overwrite=False):
# starttime is Time String
# endtime is Time String
# interval is AFTimeSpan
# desiredUnits is a uom, cannot be specified for PI points
# fiterexpression is filtering like * or SINU*
# includefilteredvalues bool: Specify 'true' to indicate that values which fail the filter criteria are present in the returned data at the times where they occurred with a value set to a 'Filtered' enumeration value with bad status. Repeated consecutive failures are omitted.
# sortorder default is 'Ascending'
# summaryDuration The duration of each summary interval. If specified in hours, minutes, seconds, or milliseconds, the summary durations will be evenly spaced UTC time intervals. Longer interval types are interpreted using wall clock rules and are time zone dependent.
# TODO: add starttime parameter
# TODO: add endtime parameter
# TODO: add boundary parameter
# TODO: add interval parameter
calctype = calculationtype.lower()
is_single_value = True if calctype == 'current' or calctype == 'end' \
else False
log.debug('Calculation type: %s, Single value: %s', calctype,
is_single_value)
for point in iterfy(points):
log.debug('Retrieving %s data for %s...', calctype, point.name)
if calctype == 'current':
payload = {'time': time}
elif calctype == 'interpolated':
payload = {'startTime': starttime, 'endTime': endtime,
'interval': interval,
'filterExpression': filterexpression,
'includeFilteredValues': includefilteredvalues}
elif calctype == 'interpolatedattimes':
payload = {'time': time, 'filterExpression': filterexpression,
'includeFilteredValues': includefilteredvalues,
'sortOrder': sortorder}
elif calctype == 'recorded':
payload = {'startTime': starttime, 'endTime': endtime,
'boundaryType': boundarytype,
'filterExpression': filterexpression,
'includeFilteredValues': includefilteredvalues,
'maxCount': maxcount}
elif calctype == 'recordedattime':
payload = {'time': time, 'retrievalMode': retrievalmode}
elif calctype == 'plot':
payload = {'startTime': starttime, 'endTime': endtime,
'intervals': intervals}
elif calctype == 'summary':
payload = {'startTime': starttime, 'endTime': endtime,
'summaryType': summarytype,
'calculationBasis': calculationbasis,
'timeType': timetype,
'summaryDuration': summaryduration,
'sampleType': sampletype,
'sampleInterval': sampleinterval,
'filterExpression': filterexpression}
elif calctype == 'end':
payload = {}
else:
log.debug('This %s request has no URL parameters', calctype)
endpoint = get_endpoint(self.url, point, calctype)
# TODO: add queryParamater generator function here?
try:
log.debug('Instantiating %s request for PI point %s to '
'endpoint %s with the following parameters: %s',
calctype, point.name, endpoint, payload)
r = self.session.get(endpoint, params=payload)
if r.status_code != requests.codes.ok:
r.raise_for_status()
except OSIsoftPyException as e:
log.error('Exception while retrieving recorded values'
'from %s for %s. Raw JSON: %s', endpoint, point.name,
exc_info=True)
data = r.json()
log.debug('HTTP %s - Instantiating OSIsoftPy.Values()',
r.status_code)
log.debug('Staging PI point value for '
'instantiation...')
try:
new_values = get_point_values(point, calctype, data)
log.debug('%s %s value(s) were instantiated for %s.',
get_count(new_values), calctype, point.name)
except OSIsoftPyException as e:
log.error('Exception while instantiating PI Point value(s)'
'for %s. Raw JSON: %s', point.name, data,
exc_info=True)
current_values = TypedList(validtypes=Value)
if is_single_value:
try:
value = getattr(point, get_attribute(calctype))
log.debug('Storing %s value.', calctype)
current_values.append(value)
except TypeError as e:
log.warning('TypeError encountered - the attribute %s is '
'empty for %s, which will raise an '
'exception when trying to iterate.',
get_attribute(calctype), point.name,
exc_info=False)
else:
try:
for value in getattr(point, get_attribute(calctype)):
log.debug(
'Storing %s value for PI point %s, attribute: %s',
calctype, point.name, get_attribute(calctype))
current_values.append(value)
except TypeError as e:
log.warning('TypeError encountered - the attribute %s is '
'empty for %s, which will raise an '
'exception when trying to iterate.',
get_attribute(calctype), point.name,
exc_info=False)
log.debug('PI point %s currently has %s %s values.', point.name,
get_count(current_values), calctype)
if is_single_value and overwrite:
log.debug('Single point value - overwriting existing %s '
'value, Single value: %s.', calctype,
is_single_value)
setattr(point, get_attribute(calctype), new_values[0])
elif is_single_value and append:
log.debug('Single point value - append is true but cannot '
'append...overwriting existing %s '
'value, Single value: %s.', calctype,
is_single_value)
setattr(point, get_attribute(calctype), new_values[0])
elif not is_single_value and overwrite:
log.debug('Multiple point values - overwriting %s existing '
'%s values, Single value: %s.',
get_count(current_values), calctype, is_single_value)
setattr(point, get_attribute(calctype), new_values)
elif not is_single_value and append:
for new_value in new_values:
current_values.append(new_value)
setattr(point, get_attribute(calctype), current_values)
else:
# TODO: allow both to be false if no data exists.
log.error('Error saving %s new %s point value(s) for PI '
'point %s. Single value: %s, Overwrite: %s, Append: '
'%s.', get_count(new_values), calctype, point.name,
is_single_value, overwrite, append)
return points
def congress_votes(congress_num):
url = vote_url + "?congress=%s" % (congress_num)
count = get_count(url)
url += "&limit=%s" % (count)
votes = dl(url)['objects']
return votes
image_datasets = {
x: Mura(mura_dataset[x], transform=data_transforms[x])
for x in data_cat
}
dataloaders = {
x: DataLoader(image_datasets[x], batch_size=1, shuffle=True, num_workers=4)
for x in data_cat
}
dataset_sizes = {x: len(mura_dataset[x]) for x in data_cat}
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# ta = total abnormal images, tn = total normal images
ta = {x: get_count(mura_dataset[x], 'positive') for x in data_cat}
tn = {x: get_count(mura_dataset[x], 'negative') for x in data_cat}
W1 = {x: np_V(tn[x] / (tn[x] + ta[x])) for x in data_cat}
W0 = {x: np_V(ta[x] / (tn[x] + ta[x])) for x in data_cat}
base_model = models.densenet169(pretrained=True)
for param in base_model.parameters():
param.requires_grad = False
in_feat = base_model.classifier.out_features
#Adding a linear layer that produces only 1 class output and applying Sigmoid over it
model = nn.Sequential(base_model, nn.Linear(in_feat, 1), nn.Sigmoid())
model = model.to(device)
# 'users': 2298698,
# 'items': 1362,
# 'interaction': 23416418,
# 'entities': 28115,
# 'relations': 7,
# 'kg_triples': 160519,
# 'hyperbolicity': 0
# },
# 'movie20m': {
# 'users': 138159,
# 'items': 16954,
# 'interaction': 13501622,
# 'entities': 102569,
# 'relations': 32,
# 'kg_triples': 499474,
# 'hyperbolicity': 0
# }
# }
print('DATASETS:' + str(DATASETS))
non_dataset = []
for dataset in DATASETS:
try:
number[dataset] = get_count(dataset)
except:
pass
if number[dataset] == None:
number.pop(dataset)
non_dataset.append(dataset)
print('NUMBER:' + str(number))
print('No-dataset: %s' % str(non_dataset))