def publish(self):
"""Publish a run to the OpenML server.
Uploads the results of a run to OpenML.
Sets the run_id on self
Returns
-------
self : OpenMLRun
"""
if self.model is None:
raise PyOpenMLError("OpenMLRun obj does not contain a model. (This should never happen.) ");
if self.flow_id is None:
raise PyOpenMLError("OpenMLRun obj does not contain a flow id. (Should have been added while executing the task.) ");
description_xml = self._create_description_xml()
file_elements = {'description': ("description.xml", description_xml)}
if self.error_message is None:
predictions = arff.dumps(self._generate_arff_dict())
file_elements['predictions'] = ("predictions.arff", predictions)
if self.trace_content is not None:
trace_arff = arff.dumps(self._generate_trace_arff_dict())
file_elements['trace'] = ("trace.arff", trace_arff)
return_value = _perform_api_call("/run/", file_elements=file_elements)
run_id = int(xmltodict.parse(return_value)['oml:upload_run']['oml:run_id'])
self.run_id = run_id
return self
def publish(self):
"""Publish a run to the OpenML server.
Uploads the results of a run to OpenML.
Sets the run_id on self
Returns
-------
self : OpenMLRun
"""
if self.model is None:
raise PyOpenMLError("OpenMLRun obj does not contain a model. (This should never happen.) ");
if self.flow_id is None:
raise PyOpenMLError("OpenMLRun obj does not contain a flow id. (Should have been added while executing the task.) ");
description_xml = self._create_description_xml()
file_elements = {'description': ("description.xml", description_xml)}
if self.error_message is None:
predictions = arff.dumps(self._generate_arff_dict())
file_elements['predictions'] = ("predictions.arff", predictions)
if self.trace_content is not None:
trace_arff = arff.dumps(self._generate_trace_arff_dict())
file_elements['trace'] = ("trace.arff", trace_arff)
return_value = _perform_api_call("/run/", file_elements=file_elements)
run_id = int(xmltodict.parse(return_value)['oml:upload_run']['oml:run_id'])
self.run_id = run_id
return self
def publish(self) -> 'OpenMLRun':
""" Publish a run (and if necessary, its flow) to the OpenML server.
Uploads the results of a run to OpenML.
If the run is of an unpublished OpenMLFlow, the flow will be uploaded too.
Sets the run_id on self.
Returns
-------
self : OpenMLRun
"""
if self.model is None:
raise PyOpenMLError("OpenMLRun obj does not contain a model. "
"(This should never happen.) ")
if self.flow_id is None:
if self.flow is None:
raise PyOpenMLError(
"OpenMLRun object does not contain a flow id or reference to OpenMLFlow "
"(these should have been added while executing the task). "
)
else:
# publish the linked Flow before publishing the run.
self.flow.publish()
self.flow_id = self.flow.flow_id
if self.parameter_settings is None:
if self.flow is None:
self.flow = openml.flows.get_flow(self.flow_id)
self.parameter_settings = self.flow.extension.obtain_parameter_values(
self.flow,
self.model,
)
description_xml = self._create_description_xml()
file_elements = {'description': ("description.xml", description_xml)}
if self.error_message is None:
predictions = arff.dumps(self._generate_arff_dict())
file_elements['predictions'] = ("predictions.arff", predictions)
if self.trace is not None:
trace_arff = arff.dumps(self.trace.trace_to_arff())
file_elements['trace'] = ("trace.arff", trace_arff)
if self.additional_information is not None:
for (name, (file,
contents)) in self.additional_information.items():
file_elements[name] = (file, contents)
return_value = openml._api_calls._perform_api_call(
"/run/", 'post', file_elements=file_elements)
result = xmltodict.parse(return_value)
self.run_id = int(result['oml:upload_run']['oml:run_id'])
return self
def toArffString(self):
data = dict()
data.update({"description": self.getDescription()})
data.update({"relation": self.relationName()})
attributes = []
for i in range(self.numAttributes()):
t = []
t.append(self.attribute(i).name())
if self.attribute(i).type() == Attribute.NUMERIC:
t.append("REAL")
else:
t.append(self.attribute(i).m_AttributeInfo.m_Values)
attributes.append(tuple(t))
data.update({"attributes": attributes})
datas = []
for i in range(self.numInstances()):
val = []
for j in range(self.numAttributes()):
if self.instance(i).isMissing(j):
val.append(None)
elif self.attribute(j).isNominal():
val.append(
self.attribute(j).value(self.instance(i).value(j)))
else:
val.append(str(self.instance(i).value(j)))
datas.append(val)
data.update({"data": datas})
text = arff.dumps(data)
return text
def store_as_arff(data,
labels,
relation,
path,
description=u'',
attributes=ATTRIBUTES):
"""
Writes feature data to an ARFF file.
:param data: The data set. (Without class labels)
:param labels: Class labels in the same order as the data instances in ``data``
:param relation: The name of the relation
:param path: The full path to the file to write to.
:param description: A description of the relation.
:param attributes: The attributes in the data set and their levels
"""
attributes.append(('@@TRUTH@@', ['Y', 'N']))
new_data = []
for i in range(len(data)):
l = list(data[i])
l += labels[i]
new_data.append(l)
arff_data = {
'description': description,
'relation': relation,
'attributes': attributes,
'data': new_data,
}
fh = open(path, 'w')
fh.write(liac_arff.dumps(arff_data))
fh.close()
def partg():
data = arff.load(open(filename))
# find the attribute index
for attribute_index in range(len(data['attributes'])):
if data['attributes'][attribute_index][0] == attribute:
break
if attribute_index == len(data['attributes']):
print("The given attribute name does not exist.")
exit(1)
attribute_type = data['attributes'][attribute_index][1]
# Add attribute 10 times
for i in range(10):
# Add a new attribute
data['attributes'].insert(0, (attribute + str(i), attribute_type))
# Append a copy to each piece of data
for j in range(len(data['data'])):
data['data'][j].insert(0, data['data'][j][attribute_index])
# Write a new file
p4g_file = open('partg-' + filename, 'w')
p4g_file.write(arff.dumps(data))
def vectors_to_arff_format():
# To be used for Weka analysis
data['data'] = read_feat()
print(arff.dumps(data))
# compute_features_vector()
def export(self, paintings):
"""Export the analysed data."""
data = {'description': self.__class__.__name__,
'relation': 'year',
'attributes': self.get_attributes(),
'data': self.get_values(paintings)}
return arff.dumps(data)
def store_as_arff(data, labels, relation, path, description=u'', attributes=ATTRIBUTES):
"""
Writes feature data to an ARFF file.
:param data: The data set. (Without class labels)
:param labels: Class labels in the same order as the data instances in ``data``
:param relation: The name of the relation
:param path: The full path to the file to write to.
:param description: A description of the relation.
:param attributes: The attributes in the data set and their levels
"""
attributes.append(('@@TRUTH@@', ['Y', 'N']))
new_data = []
for i in range(len(data)):
l = list(data[i])
l += labels[i]
new_data.append(l)
arff_data = {
'description': description,
'relation': relation,
'attributes': attributes,
'data': new_data,
}
fh = open(path, 'w')
fh.write(liac_arff.dumps(arff_data))
fh.close()
def findMissingValues(filename):
with open(filename, 'r+') as af:
arffFile = arff.load(af)
data = arffFile['data']
attributes = arffFile['attributes']
numExamples = len(data)
averages = []
# loop over each attribute
for index in range(len(attributes)):
attr = attributes[index]
average = '?'
if isinstance(attr[1], list): # find mode if attr is classifier
words_to_count = (row[index] for row in data if row[index] != None)
c = Counter(words_to_count)
average = c.most_common(1)[0][0] # stacks on stacks
else: # find mean
average = sum([row[index] for row in data if row[index] != None]) / numExamples
averages.append(average)
# udpate the missing values
for row in data:
for index in range(len(row)):
if row[index] == None:
row[index] = averages[index]
# overwrite the file
af.seek(0)
af.write(arff.dumps(arffFile))
af.truncate()
return data
def cccCentring(ra, combnk, files, aRa, rGoldIndiv):
for i in range(len(v.eName)):
for f, fname in enumerate(files[i][0]):
meanByF = []
wghRater = []
csv = rGoldIndiv[v.eName[i]][f]
#Firstly we compute the mean of all raters for each file
for a in range(v.nAn):
#We get the mean
meanRatersF = np.nanmean(csv[:, a + 1])
meanByF.append(meanRatersF)
#We take the weight of the rater in this file
wghRater.append(aRa[a][i][f])
#Now we calculate the ponderate mean of all raters
pondMean = np.sum(np.multiply(meanByF, wghRater)) / np.sum(
aRa[:, i, f])
#We have the mean of all raters, we need the total mean of the file
meanF = np.nanmean(csv[:, 1:])
#Now we will center each prediction according to the mean
output = []
#We prepare the ARFF file, we get the template
data = arff.load(open(v.arffTempPath, 'rb'))
for line in range(len(csv) - 1):
meanLine = np.nanmean(csv[line + 1, 1:])
newGs = meanLine - meanF + pondMean
#We replace the values in the ARFF template
data["data"][line][0] = fname.replace(".csv", "")
data["data"][line][1] = round(csv[line + 1, 0], 2)
data["data"][line][2] = round(newGs, 6)
#We write the csv in the Gold Standard folder
f = open(v.agsc[i] + fname.replace(".csv", ".arff"), "w")
f.write(arff.dumps(data))
return None
def splitFile(filename):
numClasses = 0
classData = dict()
with open(filename, 'rb') as af:
arffFile = arff.load(af)
attributes = arffFile['attributes']
classes = attributes[-1][1]
# replaces empty list for Firstyrcumgpa
template['attributes'][-1] = (template['attributes'][-1][0], classes)
numClasses = len(classes)
for c in classes:
classData.setdefault(c, [])
arffData = arffFile['data']
for row in arffData:
if row[-1] != None:
cList = classData[row[-1]]
cList.append(row)
# save each key of classData to a sepporate arff
filenum = 0
for key, data in classData.items():
template['data'] = data
with open(temp_dir + '\o%g.arff' % filenum, 'w') as arffFile:
arffFile.write(arff.dumps(template))
filenum += 1
return numClasses
def make_arff(symbol, preparer, attributes):
symbol_quotes = list(quotes.find({'Symbol': symbol}))
data = [preparer(instance, i, symbol_quotes)
for i, instance in enumerate(symbol_quotes[:-1])]
return arff.dumps(data, relation=RELATION_NAME % symbol,
names=attributes)
def do_run(task, optimizer, output_dir, internet_access=True, publish=False):
if internet_access:
run = openml.runs.run_model_on_task(task, optimizer)
score = run.get_metric_fn(sklearn.metrics.accuracy_score)
print('%s [SCORE] Data: %s; Accuracy: %0.2f' %
(openmlpimp.utils.get_time(), task.get_dataset().name,
score.mean()))
if publish:
run = run.publish()
run_xml = run._create_description_xml()
predictions_arff = arff.dumps(run._generate_arff_dict())
with open(output_dir + '/run.xml', 'w') as f:
f.write(run_xml)
with open(output_dir + '/predictions.arff', 'w') as f:
f.write(predictions_arff)
if run.trace_content is not None:
trace_arff = arff.dumps(run._generate_trace_arff_dict())
with open(output_dir + '/trace.arff', 'w') as f:
f.write(trace_arff)
return run
else:
res = openml.runs.functions._run_task_get_arffcontent(
optimizer, task, task.class_labels)
run = openml.runs.OpenMLRun(task_id=task.task_id,
dataset_id=None,
flow_id=None,
model=optimizer)
run.data_content, run.trace_content, run.trace_attributes, run.fold_evaluations, _ = res
score = run.get_metric_fn(sklearn.metrics.accuracy_score)
print('%s [SCORE] Data: %s; Accuracy: %0.2f' %
(openmlpimp.utils.get_time(), task.get_dataset().name,
score.mean()))
if run.trace_content is not None:
trace_arff = arff.dumps(run._generate_trace_arff_dict())
with open(output_dir + '/trace.arff', 'w') as f:
f.write(trace_arff)
predictions_arff = arff.dumps(run._generate_arff_dict())
with open(output_dir + '/predictions.arff', 'w') as f:
f.write(predictions_arff)
return run
def dumps(df):
"""
dump DataFrame to str
:param DataFrame df:
:rtype: str
:return: dumped arff
"""
arff = __dump(df)
return liacarff.dumps(arff)
def parth():
data = arff.load(open(filename))
for i in range(20):
data['attributes'].insert(0, ('RANDOM' + str(i), 'NUMERIC'))
for j in range(len(data['data'])):
data['data'][j].insert(0, random.random())
p4h_file = open('parth-' + filename, 'w')
p4h_file.write(arff.dumps(data))
def test_encode_destiny(self):
src = ARFF_DESTINY
count = 0
while count < 10:
count += 1
obj = arff.loads(src)
src = arff.dumps(obj)
self.assertEqual(src, ARFF_DESTINY)
def VTiter(self, *parsedArgs, **envars):
import arff
largs, dictargs = self.full_parse(parsedArgs)
self.nonames = True
self.names = []
self.types = []
data = {}
if 'query' not in dictargs:
raise functions.OperatorError(
__name__.rsplit('.')[-1], "No query argument ")
query = dictargs['query']
cur = envars['db'].cursor()
c = cur.execute(query)
schema = cur.getdescriptionsafe()
raw = []
try:
first_row = c.next()
raw.append(first_row)
except:
f = open('input.arff', 'w')
f.write(
"@RELATION hour-weka.filters.unsupervised.attribute.Remove-R1-2\n\n"
)
for j in schema:
f.write("@ATTRIBUTE %s NUMERIC\n" % (j[0], ))
f.write("\n@DATA\n")
yield (('result', ), )
yield (1, )
return
updated_schema = []
for i in range(len(schema)):
t = (schema[i][0], "NUMERIC")
updated_schema.append(t)
data[u'attributes'] = updated_schema
for row in c:
raw.append(row)
data[u'data'] = raw
data[u'description'] = u''
data[
u'relation'] = u'hour-weka.filters.unsupervised.attribute.Remove-R1-2'
f = open('input.arff', 'w')
f.write(arff.dumps(data))
yield (('result', ), )
yield (1, )
def run_experiment(rscv, task, args):
try:
count = 1
while count <= 100:
try:
print("%s Started classifier %s, condition %s, parameter '%s', deftype '%s', RS seed %s on task %s, dataset '%s'." % (hyperimp.utils.get_time(), args.classifier, args.condition, args.param, args.deftype, args.seed, args.task_id, task.get_dataset().name))
# train model
run = train_model(task, rscv)
break
except openml.exceptions.OpenMLServerError as e:
if count == 100:
print("%s OpenMLServerError in run, I tried this 100 times already, so I'm just going to continue to the next run." % (hyperimp.utils.get_time()))
raise
sleeptime = randint(5,60)
print("%s Error in run, trying again in %d seconds. Message: %s" % (hyperimp.utils.get_time(), sleeptime, e))
count += 1
sleep(sleeptime)
run.tags.append('study_%s' %str(args.study_id))
score = run.get_metric_fn(sklearn.metrics.accuracy_score)
print('%s [SCORE] Accuracy: %0.2f.' % (hyperimp.utils.get_time(), score.mean()))
if args.log:
# log xml, predictions
output_dir = args.output_dir + '/' + args.classifier + '/task_' + str(args.task_id) + '/' + str(args.condition)
os.makedirs(output_dir)
run_xml = run._create_description_xml()
predictions_arff = arff.dumps(run._generate_arff_dict())
with open(output_dir + '/run.xml', 'w') as f:
f.write(run_xml)
with open(output_dir + '/predictions.arff', 'w') as f:
f.write(predictions_arff)
else:
None
count_run = 1
while count_run <= 100:
try:
# publish run on OpenML
run.publish()
break
except openml.exceptions.OpenMLServerError as e:
if count_run == 100:
print("%s OpenMLServerError in run, I tried uploading this 100 times already, so I'm just going to continue to the next run." % (hyperimp.utils.get_time()))
raise
sleeptime_run = randint(5,60)
print("%s Error in uploading run trying again in %d seconds. Message: %s" % (hyperimp.utils.get_time(), sleeptime_run, e))
count_run += 1
sleep(sleeptime_run)
print("%s Uploaded run condition %s, parameter %s, RS seed %s, task %s, with run id %d." % (hyperimp.utils.get_time(), args.condition, args.param, args.seed, args.task_id, run.run_id))
except TimeoutError as e:
print("%s Run timed out." % (hyperimp.utils.get_time()))
except Exception as e:
print("%s Error in run: %s" % (hyperimp.utils.get_time(), e))
traceback.print_exc()
return
def csv_to_arff(X, label_i, savePath, datatype, isTrain=True):
# get attributes
if datatype == 'real':
attributes = [(X.columns[i], u"REAL") for i in range(len(X.columns))]
attributes.append(('label_' + label_i.name, ['0', '1']))
data = []
i = 0
while i < len(label_i):
attr_data = [j for j in list(X.iloc[i, :])]
label_data = [str(label_i[i])]
row_data = attr_data + label_data
data.append(row_data)
i += 1
# set obj
obj = {
'description': u'',
'relation': 'relation',
'attributes': attributes,
'data': data,
}
elif datatype == "nominal":
attributes = [('attr_' + X.columns[i], ['0', '1'])
for i in range(len(X.columns))]
attributes.append(('label_' + label_i.name, ['0', '1']))
data = []
i = 0
while i < len(label_i):
attr_data = [str(int(j)) for j in list(X.iloc[i, :])]
label_data = [str(label_i[i])]
row_data = attr_data + label_data
data.append(row_data)
i += 1
# set obj
obj = {
'description': u'',
'relation': 'relation',
'attributes': attributes,
'data': data,
}
else:
raise TypeError("datatype.")
arff_data = arff.dumps(obj)
if isTrain:
#w_file = open(savePath+label_i.name+"_train.arff", "w")
w_file = open(savePath + "/train.arff", "w")
w_file.write(arff_data)
w_file.close()
elif not isTrain:
w_file = open(savePath + "/test.arff", "w")
w_file.write(arff_data)
w_file.close()
else:
raise (ValueError, "what type of dataset?")
def test_simple(self):
dumps = self.get_dumps()
s = dumps(OBJ)
self.assertEqual(s, ARFF)
count = 0
while count < 10:
count += 1
obj = arff.loads(s)
src = arff.dumps(obj)
self.assertEqual(src, ARFF)
def publish(self):
"""Publish a run to the OpenML server.
Uploads the results of a run to OpenML.
"""
predictions = arff.dumps(self._generate_arff())
description_xml = self._create_description_xml()
data = {'predictions': ("predictions.csv", predictions),
'description': ("description.xml", description_xml)}
return_code, return_value = _perform_api_call(
"/run/", file_elements=data)
return return_code, return_value
def _get_file_elements(self) -> Dict:
""" Get file_elements to upload to the server.
Derived child classes should overwrite this method as necessary.
The description field will be populated automatically if not provided.
"""
if self.model is None:
raise PyOpenMLError(
"OpenMLRun obj does not contain a model. "
"(This should never happen.) "
)
if self.flow_id is None:
if self.flow is None:
raise PyOpenMLError(
"OpenMLRun object does not contain a flow id or reference to OpenMLFlow "
"(these should have been added while executing the task). "
)
else:
# publish the linked Flow before publishing the run.
self.flow.publish()
self.flow_id = self.flow.flow_id
if self.parameter_settings is None:
if self.flow is None:
self.flow = openml.flows.get_flow(self.flow_id)
self.parameter_settings = self.flow.extension.obtain_parameter_values(
self.flow,
self.model,
)
file_elements = {'description': ("description.xml", self._to_xml())}
if self.error_message is None:
predictions = arff.dumps(self._generate_arff_dict())
file_elements['predictions'] = ("predictions.arff", predictions)
if self.trace is not None:
trace_arff = arff.dumps(self.trace.trace_to_arff())
file_elements['trace'] = ("trace.arff", trace_arff)
return file_elements
def store(self, graph_builder, **kwargs):
""" Stores the graph content in Conll format into the object file.
:param graph: The graph to store.
:param kwargs: Unused
"""
import arff
data = []
entities = graph_builder.get_all_coref_entities()
self.logger.debug("entities: %s", len(entities))
for entity in entities:
mentions = graph_builder.get_all_entity_mentions(entity)
self.logger.debug("Mentions: %s", len(mentions))
for mention in mentions:
surface_learn = mention.get("surface_learn", {})
self.logger.debug("links: %s", len(surface_learn))
for link in surface_learn:
data.append(surface_learn[link])
self.logger.debug("link: %s: ", surface_learn[link])
boolean = [str(True), str(False)]
arff_file = {
'attributes': [
('relax_match', boolean),
('mention_enumeration', boolean),
('candidate_enumeration', boolean),
('mention_appositive', boolean),
('candidate_appositive', boolean),
('equal_names', 'REAL'),
('equal_adjectives', 'REAL'),
('equal_rest', 'REAL'),
('extra_mention_names', 'REAL'),
('extra_mention_adjectives', 'REAL'),
('extra_mention_rest', 'REAL'),
('extra_candidate_names', 'REAL'),
('extra_candidate_adjectives', 'REAL'),
('extra_candidate_rest', 'REAL'),
('mention', 'STRING'),
('candidate', 'STRING'),
('sentence_distance', 'INTEGER'),
('linked', boolean)],
'description': 'surface relations between mentions',
'relation': 'surface',
'data': data
}
if len(data):
self.file.write(arff.dumps(arff_file))
else:
self.logger.info("Empty data")
def export_arff(file_name: str, data: DataFrame, attributes, description: str):
if description is None:
description = ''
if attributes is None:
attributes = [(attribute, "STRING") for attribute in data.columns]
arff_dict = {
'description': description,
'relation': file_name,
'attributes': attributes,
'data': data.values.tolist()
}
return "data:text/arff;charset=utf-8," + \
urllib.parse.quote(arff.dumps(arff_dict))
def arffGenerator(self, dataList, rawNamesList, group):
namesList = []
for name in rawNamesList:
namesList.append((name,'REAL'))
arffDict = {}
arffDict['description'] = u''
arffDict['relation'] = 'perfEvents'
arffDict['attributes'] = namesList
arffDict['data'] = dataList
# print arff.dumps(arffDict)
outFile = os.path.join(self.dumpTo, group + '.arff')
with open(outFile, 'w') as f:
f.write(arff.dumps(arffDict))
def arffGenerator(self, dataList, rawNamesList, group):
namesList = []
for name in rawNamesList:
namesList.append((name, 'REAL'))
arffDict = {}
arffDict['description'] = u''
arffDict['relation'] = 'perfEvents'
arffDict['attributes'] = namesList
arffDict['data'] = dataList
# print arff.dumps(arffDict)
outFile = os.path.join(self.dumpTo, group + '.arff')
with open(outFile, 'w') as f:
f.write(arff.dumps(arffDict))
def transform(file, dont_care_category, a, attributes_file):
"""input example
[ {id: 123, text: "this is text body", category: ["dont_care"]} ]
output example
@relation game_media_bot
@attribute
:return:
"""
classes = set()
data = json.load(open(file, 'r'))
master_vector = Counter()
for tweet in data:
classes.add(tweet['category'][0])
if tweet['category'][0] != dont_care_category:
master_vector += get_word_vector(tweet)
print(master_vector)
attrs = None
if not attributes_file:
# most common words in the text of the target category
attrs = [(word, 'INTEGER') for word, _ in master_vector.most_common(a)]
attrs.append(('class', [value for value in classes]))
else:
# load attributes from this file
arff_data = arff.load(open(attributes_file, 'r'))
attrs = arff_data['attributes']
arff_data = {
'attributes': attrs,
'data': [],
'description': '',
'relation': '{}'.format(dont_care_category)
}
for tweet in data:
word_vector = get_word_vector(tweet)
tweet_data = [word_vector[attr[0]] for attr in attrs[:-1]]
tweet_data.append(tweet['category'][0])
arff_data['data'].append(tweet_data)
out_file = file.replace('.json', '.arff')
data = arff.dumps(arff_data)
with open(out_file, 'w') as f:
f.write(data)
return out_file
def md2arff():
ri_obj_list = get_repo_info(to_dict=False, combine_star_events=True)
repo_set = set()
data = dict()
data['attributes'] = attrs
data['description'] = ''
data['relation'] = 'readme'
readme_file_set = set()
inline_data = list()
for ri in ri_obj_list:
if (ri.repo_owner, ri.repo_name) in repo_set:
continue
repo_set.add((ri.repo_owner, ri.repo_name))
paper_repo_owner = getattr(ri, 'paper_repo_owner')
paper_repo_name = getattr(ri, 'paper_repo_name')
repo_path = os.path.join(conf.repo_path, paper_repo_owner, paper_repo_name)
assert os.path.exists(repo_path)
file_list = os.listdir(repo_path)
readme_path = ''
for f in file_list:
if f.lower().startswith('readme.'):
readme_path = os.path.join(repo_path, f)
break
if readme_path == '':
readme_content = ''
else:
with open(readme_path, 'r', encoding='utf-8', errors='ignore') as readme_f:
readme_content = readme_f.read()
if readme_path != '' and f.lower() == 'readme.md':
readme_content = parse_markdown(readme_content)
readme_content = readme_content.lower()
readme_content = readme_content.replace('\n', ' ')
readme_content = readme_content.replace('\"', ' ')
readme_content = readme_content.replace('\'', ' ')
inline_data_unit = list()
if ri.stars_count >= threshold:
inline_data_unit.append('popular')
else:
inline_data_unit.append('unpopular')
inline_data_unit.append(readme_content)
inline_data.append(inline_data_unit)
data['data'] = inline_data
file_content = arff.dumps(data)
arff_path = os.path.join(conf.root_path, 'text_analysis.arff')
with open(arff_path, 'w', encoding='utf-8') as f:
f.write(file_content)
def _to_filesystem(self, file_path):
"""Serialize the trace object to the filesystem.
Serialize the trace object as an arff.
Parameters
----------
file_path: str
File path where the trace arff will be stored.
"""
trace_arff = arff.dumps(self.trace_to_arff())
with open(os.path.join(file_path, 'trace.arff'), 'w') as f:
f.write(trace_arff)
def _to_filesystem(self, file_path):
"""Serialize the trace object to the filesystem.
Serialize the trace object as an arff.
Parameters
----------
file_path: str
File path where the trace arff will be stored.
"""
trace_arff = arff.dumps(self.trace_to_arff())
with open(os.path.join(file_path, 'trace.arff'), 'w') as f:
f.write(trace_arff)
def dump_dataset(features,
feature_format,
evaluations,
train_context,
format='arff',
positive_class=None):
if format == 'arff':
data = {
'attributes': [],
'data': [],
'description': '',
'relation': 'default'
}
n_features = len(features.splitlines())
for i in range(1, n_features + 1):
feature = ('f%d' % i, ['+', '-'])
data['attributes'].append(feature)
target = train_context.target_table
if not target in train_context.orng_tables:
raise Exception(
'Target table is not preloaded in memory! Please select the `dump data` parameter in the converter widget.'
)
if feature_format == 'aleph':
target_vals = ('negative', 'positive')
else:
orng_target = train_context.orng_tables[target]
target_vals = tuple(sorted(orng_target.domain.classVar.values))
class_attr = ('class', target_vals)
data['attributes'].append(class_attr)
for line in evaluations.splitlines():
values = line.strip().split()
if feature_format == 'aleph':
class_val = values[-1]
if class_val == positive_class:
values[-1] = 'positive'
else:
values[-1] = 'negative'
data['data'].append(values)
return arff.dumps(data)
elif format == 'csv':
data = ''
for line in evaluations.splitlines():
values = line.strip().split()
data = data + ','.join(values) + '\n'
return data
return 'unsupported format'
def to_filesystem(
self,
directory: str,
store_model: bool = True,
) -> None:
"""
The inverse of the from_filesystem method. Serializes a run
on the filesystem, to be uploaded later.
Parameters
----------
directory : str
a path leading to the folder where the results
will be stored. Should be empty
store_model : bool, optional (default=True)
if True, a model will be pickled as well. As this is the most
storage expensive part, it is often desirable to not store the
model.
"""
if self.data_content is None or self.model is None:
raise ValueError('Run should have been executed (and contain '
'model / predictions)')
os.makedirs(directory, exist_ok=True)
if not os.listdir(directory) == []:
raise ValueError('Output directory {} should be empty'.format(
os.path.abspath(directory)))
run_xml = self._create_description_xml()
predictions_arff = arff.dumps(self._generate_arff_dict())
# It seems like typing does not allow to define the same variable multiple times
with open(os.path.join(directory, 'description.xml'),
'w') as fh: # type: TextIO
fh.write(run_xml)
with open(os.path.join(directory, 'predictions.arff'), 'w') as fh:
fh.write(predictions_arff)
if store_model:
with open(os.path.join(directory, 'model.pkl'),
'wb') as fh_b: # type: IO[bytes]
pickle.dump(self.model, fh_b)
if self.flow_id is None:
self.flow.to_filesystem(directory)
if self.trace is not None:
self.trace._to_filesystem(directory)
def normalizeFunction(self, data, technique, folderName, arffs):
t = time.time()
# Choose Standardization Technique
if technique == 'MinMaxScaler':
scaler = MinMaxScaler()
if technique == 'StandardScaler':
scaler = StandardScaler()
if technique == 'MaxAbsScaler':
scaler = MaxAbsScaler()
if technique == 'RobustScaler':
scaler = RobustScaler()
print("Convertendo para Normalização {}..".format(technique))
print("Transformando em Dataframe..")
# Convert data into DataFrame
df = pd.DataFrame(data['data'])
# Get features number
length = df.iloc[0, :].values
print("Número de Features: {}".format(len(length) - 1))
# Create a label column
labels = df.iloc[:, len(length) - 1].values
# Save features without labels
data_aux = df.iloc[:, 0:(len(length) - 1)].values
# Normalize data
data_normalized = scaler.fit_transform(data_aux)
# Adding the labels to normalized data
data_normalized = np.concatenate((data_normalized, np.vstack(labels)),
axis=1)
# Replacing data with normalized samples
data['data'] = data_normalized
# Creating Folder if doesnt exists
try:
os.mkdir(folderName + "Normalized_Arffs", 755)
print("Criando pasta onde será salvo os arquivos arffs..")
except:
print(
"Pasta já existente, apenas sobrescrevendo os arquivos arffs.."
)
# Saving arff in text file
print("Salvando arff..")
newArffFile = open(
folderName + "Normalized_Arffs/" + arffs[:-5] + "_" + technique +
".arff", "w")
newArffFile.write(arff.dumps(data))
newArffFile.close()
print("Processo para o arquivo {} terminado. (Tempo de execução: {})".
format(arffs,
time.time() - t))
print("")
def test_files(self):
fname = os.path.join(SRC_DIR, 'example.arff')
data = [
['blonde', 17.2, 1],
['blue', 27.2, 2],
['blue', 18.2, 3],
]
arff.dump(fname, data, relation='diabetics_data', names=('hair_color', 'age', 'patno'))
data = list(arff.load(os.path.join(SRC_DIR, fname)))
arff_rows = arff.dumps(data)
reparsed_data = list(arff.loads(arff_rows))
data = [list(row) for row in data]
reparsed_data = [list(row) for row in reparsed_data]
self.assertEqual(data, reparsed_data)
def generate(arff_file):
ou = open(arff_file, "w")
dataset = {
'description': 'Motion sensor dataset',
'relation': 'whatever',
'attributes': [
('chair_prev', 'REAL'),
('bath_prev', 'REAL'),
('down_prev', 'REAL'),
('up_prev', 'REAL'),
('chair_post', 'REAL'),
('bath_post', 'REAL'),
('down_post', 'REAL'),
('up_post', 'REAL'),
('a_prev', 'REAL'),
('a_post', 'REAL'),
('tag', ['walk', 'chair', 'bath', 'down', 'up'])
]
}
sql = """select * from motion order by event_timestamp asc;"""
m.execute(sql)
data = []
counter = 0
for record in m:
# print(record)
row = []
ts = float(record[5])
prev = get_prev_obj(ts)
post = get_post_obj(ts)
for item in prev:
row.append(item)
for item in post:
row.append(item)
a = get_a(ts)
for item in a:
row.append(item)
if record[-1] is None:
row.append('?')
else:
row.append(record[-1])
data.append(row)
counter += 1
print(counter)
dataset['data'] = data
ou.write(arff.dumps(dataset))
ou.close()
def write_cv(self, splits):
'''
write cv.arff
'''
content = {
"relation":
"CV_Folds",
"attributes": [("instance_id", "STRING"),
("repetition", "NUMERIC"), ("fold", "NUMERIC")],
"data": []
}
for split, idx in zip(splits, range(1, self.FOLDS + 1)):
for inst_name in split:
content["data"].append([inst_name, 1, idx])
print(arff.dumps(content))
def to_filesystem(self, output_directory, store_model=True):
"""
The inverse of the from_filesystem method. Serializes a run
on the filesystem, to be uploaded later.
Parameters
----------
output_directory : str
a path leading to the folder where the results
will be stored. Should be empty
store_model : bool
if True, a model will be pickled as well. As this is the most
storage expensive part, it is often desirable to not store the
model.
"""
if self.data_content is None or self.model is None:
raise ValueError(
'Run should have been executed (and contain model / predictions)'
)
try:
os.makedirs(output_directory)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
raise e
if not os.listdir(output_directory) == []:
raise ValueError('Output directory should be empty')
run_xml = self._create_description_xml()
predictions_arff = arff.dumps(self._generate_arff_dict())
with open(os.path.join(output_directory, 'description.xml'), 'w') as f:
f.write(run_xml)
with open(os.path.join(output_directory, 'predictions.arff'),
'w') as f:
f.write(predictions_arff)
if store_model:
with open(os.path.join(output_directory, 'model.pkl'), 'wb') as f:
pickle.dump(self.model, f)
if self.trace is not None:
self.trace._to_filesystem(output_directory)
def write_cv(self, splits):
'''
write cv.arff
'''
content = {"relation" : "CV_Folds",
"attributes": [
("instance_id", "STRING"),
("repetition", "NUMERIC"),
("fold", "NUMERIC")
],
"data": []
}
for split, idx in zip(splits, range(1, self.FOLDS+1)):
for inst_name in split:
content["data"].append([inst_name, 1, idx])
print(arff.dumps(content))
def dump_dataset(features, feature_format, evaluations, train_context,
format='arff',
positive_class=None):
if format == 'arff':
data = {
'attributes': [],
'data': [],
'description': '',
'relation': 'default'
}
n_features = len(features.splitlines())
for i in range(1, n_features + 1):
feature = ('f%d' % i, ['+', '-'])
data['attributes'].append(feature)
target = train_context.target_table
if not target in train_context.orng_tables:
raise Exception('Target table is not preloaded in memory! Please select the `dump data` parameter in the converter widget.')
if feature_format == 'aleph':
target_vals = ('negative', 'positive')
else:
orng_target = train_context.orng_tables[target]
target_vals = tuple(sorted(orng_target.domain.classVar.values))
class_attr = ('class', target_vals)
data['attributes'].append(class_attr)
for line in evaluations.splitlines():
values = line.strip().split()
if feature_format == 'aleph':
class_val = values[-1]
if class_val == positive_class:
values[-1] = 'positive'
else:
values[-1] = 'negative'
data['data'].append(values)
return arff.dumps(data)
elif format == 'csv':
data = ''
for line in evaluations.splitlines():
values = line.strip().split()
data = data + ','.join(values) + '\n'
return data
return 'unsupported format'
def to_filesystem(self, output_directory, store_model=True):
"""
The inverse of the from_filesystem method. Serializes a run
on the filesystem, to be uploaded later.
Parameters
----------
output_directory : str
a path leading to the folder where the results
will be stored. Should be empty
store_model : bool
if True, a model will be pickled as well. As this is the most
storage expensive part, it is often desirable to not store the
model.
"""
if self.data_content is None or self.model is None:
raise ValueError('Run should have been executed (and contain model / predictions)')
try:
os.makedirs(output_directory)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
raise e
if not os.listdir(output_directory) == []:
raise ValueError('Output directory should be empty')
run_xml = self._create_description_xml()
predictions_arff = arff.dumps(self._generate_arff_dict())
with open(os.path.join(output_directory, 'description.xml'), 'w') as f:
f.write(run_xml)
with open(os.path.join(output_directory, 'predictions.arff'), 'w') as f:
f.write(predictions_arff)
if store_model:
with open(os.path.join(output_directory, 'model.pkl'), 'wb') as f:
pickle.dump(self.model, f)
if self.trace is not None:
self.trace._to_filesystem(output_directory)
def export_arff(file, export_file, conf):
"""Takes a JSON list of incidents, processes them, and exports an ARFF file"""
conf = json.load(open(conf))
data = prepare.prepare(file, conf)
# Structure of export
export_data = {
'attributes': [],
'data': [],
'relation': 'TrafficData'
}
paths = [jp_parse(path[1]) for path in conf['attributes']]
# print data[0]
for incident in data:
entry = []
for path in paths:
results = path.find(incident)
if len(results) > 0:
if isinstance(results[0].value, basestring):
entry.append(results[0].value.replace(" ", "-"))
else:
entry.append(results[0].value)
else:
entry.append(None)
export_data['data'].append(entry)
for index,attr in enumerate(conf['attributes']):
if attr[2] in ['NUMERIC', 'REAL', 'INTEGER']:
export_data['attributes'].append((attr[0], attr[2]))
else:
# Otherwise assume discrete
vals = list(set([incident[index] for incident in export_data['data']]))
export_data['attributes'].append((attr[0], vals))
export_data['description'] = "\n".join(
[str(datetime.datetime.now()),
json.dumps(conf, indent=2, separators=(',', ': '))])
with open(export_file, "w") as f:
f.write(arff.dumps(export_data))
def save_to_arff(cls, X, y, endian = "big", save_sparse = False):
X = X.todok()
y = y.todok()
x_prefix = 0
y_prefix = 0
x_attributes = [(u'X{}'.format(i),u'NUMERIC') for i in xrange(X.shape[1])]
y_attributes = [(u'y{}'.format(i), [unicode(0),unicode(1)]) for i in xrange(y.shape[1])]
if endian == "big":
y_prefix = X.shape[1]
relation_sign = -1
attributes = x_attributes + y_attributes
elif endian == "little":
x_prefix = y.shape[1]
relation_sign = 1
attributes = y_attributes + x_attributes
else:
raise ValueError("Endian not in {big, little}")
if save_sparse:
data = [{} for r in xrange(X.shape[0])]
else:
data = [[0 for c in xrange(X.shape[1] + y.shape[1])] for r in xrange(X.shape[0])]
for keys, value in X.iteritems():
data[keys[0]][x_prefix + keys[1]] = value
for keys, value in y.iteritems():
data[keys[0]][y_prefix + keys[1]] = value
dataset = {
u'description': u'traindata',
u'relation': u'traindata: -C {}'.format(y.shape[1] * relation_sign),
u'attributes': attributes,
u'data': data
}
return arff.dumps(dataset)
def ARFFCreation():
dataSet = ARFFDataset(Stripped, nonStripped)
attList = [
('Gazetteer', ['TRUE', 'FALSE']),
('CapitalLetter', ['TRUE', 'FALSE']),
('Preposition', ['TRUE', 'FALSE']),
('FollowingWord', ['TRUE', 'FALSE']),
('Place', ['yes', 'no'])
]
obj = {
'description': u'',
'relation': 'PlaceNames',
'attributes': attList,
'data': dataSet,
}
with open('.\CreatedCSVs\TestData.arff', 'a') as f:
f.write(arff.dumps(obj))
def save_to_arff(cls, X, y, endian="little", save_sparse=True):
"""Method for loading ARFF files as numpy array
Parameters
----------
filename : string
Path to ARFF file
labelcount: integer
Number of labels in the ARFF file
endian: string{"big", "little"}
Whether the ARFF file contains labels at the beginning of the attributes list ("big" endianness, MEKA format)
or at the end ("little" endianness, MULAN format)
input_feature_type: numpy.type as string
The desire type of the contents of the return 'X' array-likes, default 'i8',
should be a numpy type, see http://docs.scipy.org/doc/numpy/user/basics.types.html
encode_nominal: boolean
Whether convert categorical data into numeric factors - required for some scikit classifiers that can't handle non-numeric input featuers.
save_sparse: boolean
Whether to read arff file as a sparse file format, liac-arff breaks if sparse reading is enabled for non-sparse ARFFs.
Returns
-------
data: dictionary {'X': scipy sparse matrix with input_feature_type elements, 'y': scipy sparse matrix of binary (int8) label vectors }
The dictionary containing the data frame, with 'X' key storing the input space array-like of input feature vectors
and 'y' storing labels assigned to each input vector, as a binary indicator vector (i.e. if 5th position has value 1
then the input vector has label no. 5)
"""
X = X.todok()
y = y.todok()
x_prefix = 0
y_prefix = 0
x_attributes = [(u'X{}'.format(i), u'NUMERIC')
for i in xrange(X.shape[1])]
y_attributes = [(u'y{}'.format(i), [unicode(0), unicode(1)])
for i in xrange(y.shape[1])]
if endian == "big":
y_prefix = X.shape[1]
relation_sign = -1
attributes = x_attributes + y_attributes
elif endian == "little":
x_prefix = y.shape[1]
relation_sign = 1
attributes = y_attributes + x_attributes
else:
raise ValueError("Endian not in {big, little}")
if save_sparse:
data = [{} for r in xrange(X.shape[0])]
else:
data = [[0 for c in xrange(X.shape[1] + y.shape[1])]
for r in xrange(X.shape[0])]
for keys, value in X.iteritems():
data[keys[0]][x_prefix + keys[1]] = value
for keys, value in y.iteritems():
data[keys[0]][y_prefix + keys[1]] = value
dataset = {
u'description': u'traindata',
u'relation': u'traindata: -C {}'.format(y.shape[1] * relation_sign),
u'attributes': attributes,
u'data': data
}
return arff.dumps(dataset)
if(surgeryType != ""):
surgeryCount += 1
elif(radiationType != ""):
radiationCount += 1
behavior = line[223] # BEHAVIOR RECODE FOR ANALYSIS
histology = line[225:227] # HISTOLOGY RECODE - BROAD GROUPINGS
causeOfDeath = line[271] # SEER OTHER CAUSE OF DEATH CLASS
survivalMonths = line[305:309] # Survival months - presumed alive
data['data'].append([type, maritalStatus, race, tumorSize, survivalMonths])
# Write arff
arffContents = arff.dumps(data)
output = open('cancer.arff', 'w')
output.write(arffContents)
# Count things
print "Number of records: " + str(numLines)
print "Number of patients: " + str(len(patientIDs))
numRecordCounts = {}
for patientID in patientIDs :
if(patientIDs[patientID] in numRecordCounts):
numRecordCounts[patientIDs[patientID]] += 1
else:
numRecordCounts[patientIDs[patientID]] = 1
print "Number of patients with X number of records: "
def hbp_submit_search_criteria(input_dict):
import orange
import tempfile
import arff
import os
dset = arff.load(open(os.path.dirname(os.path.abspath(__file__)) + "/new_adni.arff", "rb"))
b = {}
query = input_dict["query"]
b["attributes"] = []
if query["classification"] != None:
b["attributes"].append(("Classification", [u"CN", u"AD", u"LMCI", u"EMCI", u"SMC"]))
else:
query["classification"] = [None]
if query["geo"] != None:
b["attributes"].append(
("geo", [u"Africa", u"Asia", u"Australia", u"Europe", u"North America", u"South America"])
)
else:
query["geo"] = [None]
if query["age"] != None:
b["attributes"].append(("age", [u"1-18", u"18-24", u"25-34", u"35-44", u"45-54", u"55-64", u"65+"]))
else:
query["age"] = [None]
fields = []
for c in query["classification"]:
for g in query["geo"]:
for a in query["age"]:
fields.append((c, g, a))
b["attributes"].append(("count", "NUMERIC"))
b["description"] = ""
b["relation"] = "HBP"
b["data"] = []
for field in fields:
d = []
if field[0] != None:
d.append(field[0])
if field[1] != None:
d.append(field[1])
if field[2] != None:
d.append(field[2])
counts = 0
for i in dset["data"]:
if (
(i[-1] == field[0] or field[0] == None)
and (i[0] == field[2] or field[2] == None)
and (i[1] == field[1] or field[1] == None)
):
counts = counts + 1
d.append(counts)
b["data"].append(d)
# a = arff.load_data()
f = tempfile.NamedTemporaryFile(delete=False, suffix=".arff")
f.write(arff.dumps(b))
f.close()
output_dict = {}
output_dict["results"] = orange.ExampleTable(f.name)
return output_dict
def handle(self, *args, **options):
params = json.load(open(args[0]))
source_name = params['database']
label_table_name = None
label_column_name = None
for key,value in params['label'].iteritems():
label_table_name = key
label_column_name = value
# Add parameters for setting ranges...
past = datetime.datetime(datetime.MINYEAR, 1, 1)
future = datetime.datetime(datetime.MAXYEAR, 12, 31)
for ds in DataSource.objects.filter(name__contains=source_name):
table_names = ds.table_names()
table_columns = {}
for name in table_names:
table_columns[name] = ds.table_columns(name)
# print(json.dumps(table_columns, indent=2))
points = ds.fetch_data(label_table_name, label_column_name, past, future)
label_value_name = label_table_name + '_' + label_column_name
categorical_values = {}
rows = []
row_keys = []
for point in points:
row_dict = {}
point_time = point[0]
label_value = point[1]
row_dict[label_value_name] = label_value
for table, columns in table_columns.iteritems():
fetched = ds.fetch_nearest(point_time, table, columns)
if len(fetched) > 0:
for i in range(0, len(columns)):
column = columns[i]
column_key = table + '_' + column[0]
if row_keys.count(column_key) == 0:
row_keys.append(column_key)
if column[1] == 'text' or column[1] == 'boolean':
column_values = set(['?'])
try:
column_values = categorical_values[column_key]
except KeyError:
categorical_values[column_key] = column_values
column_values.add(slugify(unicode(str(fetched[i]))))
if fetched[i] == True or fetched[i] == False:
row_dict[column_key] = slugify(unicode(str(fetched[i])))
elif column[1] == 'text':
row_dict[column_key] = slugify(unicode(fetched[i]))
else:
row_dict[column_key] = fetched[i]
rows.append(row_dict)
row_keys.sort()
data = { 'relation': label_table_name + '_' + label_column_name, 'description': '' }
attributes = []
ignore = []
for row_key in row_keys:
value_def = 'REAL'
if row_key in categorical_values:
value_def = []
for value in categorical_values[row_key]:
value_def.append(value)
if value_def == 'REAL' or len(value_def) > 1:
attributes.append((row_key, value_def))
else:
ignore.append(row_key)
data['attributes'] = attributes
data_rows = []
for row_dict in rows:
data_row = []
for row_key in row_keys:
if ignore.count(row_key) == 0:
try:
data_row.append(row_dict[row_key])
except KeyError:
data_row.append(None)
data_rows.append(data_row)
data['data'] = data_rows
print('dumping....')
print(arff.dumps(data))
print('done.')
def write_to_arff(data, filename):
with open(filename, 'w') as f:
f.write(arff.dumps(data))
def main():
lno=1
word_tot=0
corr=0
init_corr=0
beng_statistics = {}
beng_statistics = ng.ngram_prof("./beng_train.txt",beng_statistics)
eng_statistics = {}
'''
text = " ".join(brown.words())
tokenizer = RegexpTokenizer("[a-zA-Z'`]+")
text = tokenizer.tokenize(text)
text = " ".join(text)
brown_words=open("./brown_words.txt",'w')
brown_words.write(text)
brown_words.close()
'''
eng_statistics = ng.ngram_prof("./brown_words.txt",eng_statistics)
lang_stats={}
lang_stats.update({'e':eng_statistics})
lang_stats.update({'b':beng_statistics})
#fin=open("./beng_corpus.txt",'r')
fin=open("./BanglaEnglish_FIRE2013_AnnotatedDev.txt",'r')
#fout_pred=open("./predicted_tags_arff.txt",'w')
#fout_corr=open("./corrected_tags_arff.txt",'w')
word_list=[]
eng_dic=[]
beng_dic=[]
ngram=[]
surround=[]
corr_tag=[]
data=[]
sent=fin.readline()
while(sent):
############ Only for Facebook corpus (COMMENT for FIRE)
#sent = re.sub(r'[^\w\s]','',sent)
##########################################
words=[]
sent=sent.split()
for elem in sent:
############ Only for FIRE CORPUS (COMMENT for Facebook)
elem=elem.split('\\')
corr_tag.append(elem[1][0])
elem=elem[0]
#################################
elem.strip()
words.append(elem)
type_map = defaultdict(str)
type_count = defaultdict(int)
word_count=0
for word in words:
word=word.strip(" ")
word_list.append(word)
word_count+=1
beng_rat=0
eng_rat=0
if(beng.beng_word(word)):
beng_rat=1
if(eng.eng_search(word)):
eng_rat=1
############ REVERT 1/6
#eng_dic.append(eng_rat)
#beng_dic.append(beng_rat)
##############
word_statistics={}
lang_ratio = {}
grams = ngrams(word,4,pad_left=True,pad_right=True,left_pad_symbol=' ',right_pad_symbol=' ')
grams=list(grams)
ng_list=[]
for j in range (len(grams)):
ng_list.append(''.join(grams[j]))
word_statistics = ng.ngram_hash(ng_list,word_statistics)
word_statistics = sorted(word_statistics.items(), key=operator.itemgetter(1), reverse=True)
for lang, ngrams_statistics in lang_stats.items():
distance = ng.compare_ng_prof(ngrams_statistics,word_statistics)
lang_ratio.update({lang:distance})
ng_lang_result = min(lang_ratio, key = lang_ratio.get).upper()
ngram.append(ng_lang_result)
if(beng_rat and not eng_rat):
############# REVERT 2/6
beng_dic.append('2')
###################
type_word="Bengali word"
elif(eng_rat and not beng_rat):
############# REVERT 3/6
beng_dic.append('0')
###################
type_word="English word"
else:
############# REVERT 4/6
beng_dic.append('1')
###################
if(ng_lang_result=='B'):
type_word="Bengali word"
else:
type_word="English word"
type_map[word]=type_word
type_count[type_word]+=1
#print(str(word)+"(Detect:"+str(type_map[word])+")")
if((type_count["English word"])>(type_count["Bengali word"])):
default="e"
else:
default="b"
print(str(lno)+default)
lno+=1
for i in range(len(words)):
word_count=0
type_count["English word"]=0
type_count["Bengali word"]=0
if(i>1 and i<(len(words)-2)):
word_count=4
for j in range(i-2,min((len(words)-1),(i+3))):
if(j!=i):
type_count[type_map[words[j]]]+=1
elif (i<=1):
word_count=min(i+2,len(words)-1)
for j in range(min((len(words)-1),(i+3))):
if(j!=i):
type_count[type_map[words[j]]]+=1
elif(i>=(len(words)-2)):
word_count= (len(words)-i)+1
for j in range(i-2,len(words)):
if(j!=i):
type_count[type_map[words[j]]]+=1
if(word_count):
surround.append(str(type_count["Bengali word"]/word_count))
#if((type_count["Bengali word"]/word_count)>=0.5):
# surround.append('1')
#else:
# surround.append('0')
else:
surround.append('0')
sent=fin.readline()
############ Only for Facebook corpus (COMMENT for FIRE)
#real_tag = open("./real_tags.txt",'r')
#line = real_tag.readline().split()
#for word in line:
# tags=word.split('\\')
# corr_tag.append(tags[1])
#real_tag.close()
######################################
for i in range(len(word_list)):
vector=[]
# vector.append(word_list[i])
########REVERT 5/6
#vector.append(int(eng_dic[i]))
##############
vector.append(int(beng_dic[i]))
vector.append(ngram[i])
vector.append(float(surround[i]))
vector.append(corr_tag[i])
data.append(vector)
obj={
'description': u'',
'relation': 'langid',
'attributes': [
#############REVERT 6/6
#('eng_dict','NUMERIC'),
####################
('beng_dict','NUMERIC'),
('ngram','STRING'),
('beng_surr','REAL'),
('real_tag','STRING')
],
'data':data,
}
final = arff.dumps(obj)
#FIRE
final_file=open("./lang_id_fire.arff",'w')
#
#Facebook
#final_file=open("./lang_id.arff",'w')
#
final_file.write(final)
final_file.close()
fin.close()
import arff
import pickle
attributes = []
nFFT = 64
baseStr = "FFT_"
for i in range(nFFT):
name = baseStr + str(i)
attributes.append((name,u'REAL'))
attributes.append(("Ang",u'REAL'))
#print(attributes)
relation = "Dist_FFT"
dataF = []
samples = 120
for i in range(samples):
filename = str(i+1) + ".p"
f = open(filename,"rb")
p = pickle.load(f)
f.close()
p.pop(-2)
dataF.append(p)
data = {u'attributes': attributes, u'data': dataF, u'description': u'', u'relation': relation}
f = open("myData2.arff","w")
f.write(arff.dumps(data))
f.close()
def test_encode_source(self):
obj = arff.loads(ARFF_SOURCE)
result = arff.dumps(obj)
expected = ARFF_DESTINY
self.assertEqual(result, expected)
def save_to_arff(X, y, label_location="end", save_sparse=True, filename=None):
"""Method for dumping data to ARFF files
Parameters
----------
X : `array_like`, :class:`numpy.matrix` or :mod:`scipy.sparse` matrix, shape=(n_samples, n_features)
input feature matrix
y : `array_like`, :class:`numpy.matrix` or :mod:`scipy.sparse` matrix of `{0, 1}`, shape=(n_samples, n_labels)
binary indicator matrix with label assignments
label_location: string {"start", "end"} (default is "end")
whether the ARFF file will contain labels at the beginning of the
attributes list ("start", MEKA format)
or at the end ("end", MULAN format)
save_sparse: boolean
Whether to save in ARFF's sparse dictionary-like format instead of listing all
zeroes within file, very useful in multi-label classification.
filename : str or None
Path to ARFF file, if None, the ARFF representation is returned as string
Returns
-------
str or None
the ARFF dump string, if filename is None
"""
X = X.todok()
y = y.todok()
x_prefix = 0
y_prefix = 0
x_attributes = [(u'X{}'.format(i), u'NUMERIC')
for i in range(X.shape[1])]
y_attributes = [(u'y{}'.format(i), [str(0), str(1)])
for i in range(y.shape[1])]
if label_location == "end":
y_prefix = X.shape[1]
relation_sign = -1
attributes = x_attributes + y_attributes
elif label_location == "start":
x_prefix = y.shape[1]
relation_sign = 1
attributes = y_attributes + x_attributes
else:
raise ValueError("Label location not in {start, end}")
if save_sparse:
data = [{} for r in range(X.shape[0])]
else:
data = [[0 for c in range(X.shape[1] + y.shape[1])]
for r in range(X.shape[0])]
for keys, value in list(X.items()):
data[keys[0]][x_prefix + keys[1]] = value
for keys, value in list(y.items()):
data[keys[0]][y_prefix + keys[1]] = value
dataset = {
u'description': u'traindata',
u'relation': u'traindata: -C {}'.format(y.shape[1] * relation_sign),
u'attributes': attributes,
u'data': data
}
arff_data = arff.dumps(dataset)
if filename is None:
return arff_data
with open(filename, 'w') as fp:
fp.write(arff_data)
def hbp_interactive_analysis_post(postdata, input_dict, output_dict):
import orange
import tempfile
import arff
import os
import numpy
d = input_dict["Dataset"]
dset = arff.load(open(os.path.dirname(os.path.abspath(__file__)) + "/new_adni.arff", "rb"))
b = {}
b["attributes"] = []
feature_names = [x.name for x in d.domain]
if "Classification" in feature_names:
b["attributes"].append(("Classification", [u"CN", u"AD", u"LMCI", u"EMCI", u"SMC"]))
if "geo" in feature_names:
b["attributes"].append(
("geo", [u"Africa", u"Asia", u"Australia", u"Europe", u"North America", u"South America"])
)
if "age" in feature_names:
b["attributes"].append(("age", [u"1-18", u"18-24", u"25-34", u"35-44", u"45-54", u"55-64", u"65+"]))
for v in postdata["variables"]:
b["attributes"].append((v + "_avg", "NUMERIC"))
b["attributes"].append((v + "_stdev", "NUMERIC"))
b["attributes"].append(("count", "NUMERIC"))
b["description"] = ""
b["relation"] = "HBP"
b["data"] = []
for i in d:
new_i = []
if "Classification" in feature_names:
clas = i[feature_names.index("Classification")].value
new_i.append(clas)
else:
clas = None
if "geo" in feature_names:
geo = i[feature_names.index("geo")].value
new_i.append(geo)
else:
geo = None
if "age" in feature_names:
age = i[feature_names.index("age")].value
new_i.append(age)
else:
age = None
sums = {}
for v in postdata["variables"]:
sums[v] = 0
vcounts = {}
for v in postdata["variables"]:
vcounts[v] = 0
vvalues = {}
for v in postdata["variables"]:
vvalues[v] = []
count = 0
attrs = [x[0] for x in dset["attributes"]]
for ins in dset["data"]:
if (ins[-1] == clas or clas == None) and (ins[0] == geo or geo == None) and (ins[1] == age or age == None):
count = count + 1
for k, v in sums.items():
try:
sums[k] = v + ins[attrs.index(k)]
vcounts[k] = vcounts[k] + 1
vvalues[k].append(v)
except:
pass
for v in postdata["variables"]:
print vvalues[v]
stdev = numpy.std(vvalues[v])
try:
avg = sums[v] / vcounts[v]
except:
avg = -2
new_i.append(avg)
new_i.append(stdev)
new_i.append(count)
b["data"].append(new_i)
f = tempfile.NamedTemporaryFile(delete=False, suffix=".arff")
f.write(arff.dumps(b))
f.close()
output_dict = {}
output_dict["results"] = orange.ExampleTable(f.name)
return output_dict
def csv_to_arff(fileinput, type_list, relation_name, selected_attrs):
with open(fileinput, 'r') as inputfile:
data = csv.reader(inputfile, delimiter=',')
arff_content = data_to_dict(data, type_list, relation_name, selected_attrs)
return arff.dumps(arff_content)
#(u'Firstyrcumgpa', u'REAL')
]
if isTestSet:
template_attr = []
# Creat ARFF Template
template = {
u'attributes': template_attr,
u'data': data, # list
u'description': u'',
u'relation': u'admission_stats'
}
# Save ARFF FILE
with open(output_arffFile, 'w') as arffFile:
arffFile.write(arff.dumps(template))
print 'Finished Classifying...'
input_arff = output_arffFile
if not useConverts:
with open(input_arff, 'r+') as af:
arffFile = arff.load(af)
data = arffFile['data']
datalist = []
for row in data:
replaceByConversion(row)
datalist.append(row)
print 'Finished Converting...'
arffFile['data'] = datalist
def handle(self, *args, **options):
count = 0
for job in ReportJob.objects.filter(job_start=None):
params = json.loads(job.parameters)
source_name = params['database']
label_table_name = None
label_column_name = None
for key,value in params['label'].iteritems():
label_table_name = key
label_column_name = value
job.job_start = datetime.datetime.now()
job.save()
# Add parameters for setting ranges...
past = datetime.datetime(datetime.MINYEAR, 1, 1)
future = datetime.datetime(datetime.MAXYEAR, 12, 31)
for ds in DataSource.objects.filter(name__contains=source_name):
table_names = ds.table_names()
table_columns = {}
for name in table_names:
table_columns[name] = ds.table_columns(name)
points = ds.fetch_data(label_table_name, label_column_name, past, future)
label_value_name = label_table_name + '_' + label_column_name
categorical_values = {}
rows = []
row_keys = []
for point in points:
row_dict = {}
point_time = point[0]
label_value = point[1]
row_dict[label_value_name] = label_value
for table, columns in table_columns.iteritems():
fetched = ds.fetch_nearest(point_time, table, columns)
if len(fetched) > 0:
for i in range(0, len(columns)):
column = columns[i]
column_key = slugify(table + '_' + column[0])
if row_keys.count(column_key) == 0:
row_keys.append(column_key)
if column[1] == 'text' or column[1] == 'boolean':
column_values = set([None])
try:
column_values = categorical_values[column_key]
except KeyError:
categorical_values[column_key] = column_values
str_value = slugify(unicode(str(fetched[i])))
if str_value.strip() == '':
str_value = 'empty_string'
column_values.add(str_value)
if fetched[i] == True or fetched[i] == False:
row_dict[column_key] = slugify(unicode(str(fetched[i])))
elif column[1] == 'text':
str_value = slugify(unicode(fetched[i]))
if str_value.strip() == '':
str_value = 'empty_string'
row_dict[column_key] = str_value
else:
row_dict[column_key] = fetched[i]
rows.append(row_dict)
row_keys.sort()
data = { 'relation': slugify(label_table_name + '_' + label_column_name), 'description': '' }
attributes = []
ignore = []
for row_key in row_keys:
value_def = 'REAL'
if row_key in categorical_values:
value_def = []
for value in categorical_values[row_key]:
value_def.append(value)
if ('_sensor_dt_' in row_key) == False and ('randomnoiseprobe' in row_key) == False and (value_def == 'REAL' or len(value_def) > 2 or (((None in value_def) == False) and len(value_def) > 1)):
attributes.append((row_key, value_def))
else:
ignore.append(row_key)
data['attributes'] = attributes
data_rows = []
for row_dict in rows:
data_row = []
for row_key in row_keys:
if ignore.count(row_key) == 0:
try:
data_row.append(row_dict[row_key])
except KeyError:
data_row.append(None)
data_rows.append(data_row)
data['data'] = data_rows
job.result_file.save(str(job.pk) + '_' + str(uuid.uuid4()), ContentFile(arff.dumps(data)))
job.result_type = 'text/arff'
job.job_end = datetime.datetime.now()
job.save()
count += 1
print(str(count) + ' job(s) run.')