def LoadPDF(path, in_memory=False):
''' this is special function for loading images from pdf file
it actuall creates png images of pages on disk! requires WAND and ImageMagik'''
try:
from wand.image import Image #another local import to avoid proliferation of auxiliary dependencies
except:
print(
"ERROR:Can not import Wand library, that is needed for reading PDF. Please make sure wand http://docs.wand-py.org/ is installed"
)
return
import neuthink.metagraph as m # can't import before, will get circular import
diag = path
graph = Graph()
image_list: m.dNodeList = m.dNodeList(graph)
#if you see not authorized error, run this from console: sudo sed -i '/PDF/s/none/read|write/' /etc/ImageMagick-6/policy.xml
#also see here https://stackoverflow.com/questions/42928765/convertnot-authorized-aaaa-error-constitute-c-readimage-453
with (Image(filename=diag, resolution=300)) as source:
images = source.sequence
pages = len(images)
for i in range(pages):
Image(images[i]).save(filename=str(i) + '.png')
image = mvImage(path=str(i) + '.png', in_memory=in_memory)
imnode = Node(graph, {'image': image, 'type': 'image'})
image_list.append(imnode)
return image_list
def Load(path: str,
in_memory=False,
patterns=['png', 'jpg', 'jpeg'],
target_class='target_class'):
import neuthink.metagraph as m # can't import before, will get circular import
def make_list(image_names, image_list, patterns):
for iname in image_names:
# print(iname)
images, image_class = iname
if len(image_class) > 0:
image_class = image_class.split('/')[-2]
#image_name = image_class.
image_list = image_list + [
Node(
graph, {
"image": mvImage(x, in_memory=in_memory),
target_class: image_class.lower(),
'type': 'image'
}) for x in images
]
return image_list
folders = glob.glob(path + '/*/')
graph = Graph()
# print(folders)
if len(folders) > 0:
image_names = reduce(lambda x, y: x + y,
[[(glob.glob(x + '*.' + _type), x)
for x in folders] for _type in patterns])
# print(image_names)
image_list: m.dNodeList = m.dNodeList(graph)
image_list = make_list(image_names, image_list, patterns)
random.shuffle(image_list)
else:
# print("here")
image_names = reduce(lambda x, y: x + y,
[[(glob.glob(path + '/*.' + _type), "")]
for _type in patterns])
image_list: m.dNodeList = m.dNodeList(graph)
image_list = make_list(image_names, image_list, patterns)
image_list.last_call = "image"
return image_list
def LoadPostgressSQLDumpTable(filename: str,
tablename: str,
maxlines=None,
display=True,
zipped=False,
start_from=0):
'''Loads SQL dump (postgress) into nodelist (loadds singe table specified by tablename)'''
graph = Graph()
import neuthink.metagraph as m
lines: m.dNodeList = m.dNodeList(graph)
if zipped:
f = gzip.open(filename, mode='rt')
else:
f = open(filename)
state = 0
fields_list: List[str] = []
line_count = 0
for line in f:
#locate table struct line
if display:
print("Lines loaded: " + str(len(lines)) + " ",
end="\r",
flush=True)
if state == 1: #we are reading table
# if "COPY" in line: #newtable started
# state=2
values = line.split('\t')
line_count = line_count + 1
if line_count < start_from:
continue
if len(values) != len(fields_list):
state = 2 #table ended or corrputed
print()
_node = {
fields_list[i].strip(): values[i].strip()
for i in range(len(values))
}
node = Node(graph, _node)
lines.append(node)
if maxlines is not None and len(lines) > maxlines:
break
if state == 0 and tablename in line and (not '--' in line):
fields_list = line[line.index('('):line.index(')')].split(',')
# print(len(fields_list))
state = 1
print()
return lines
def MakeSubimages(self, size: int, stride: int, source='image'):
''' makes new nodelist that consists of subimages of all image'''
import neuthink.metagraph as m # can't import before, will get circular import
if source is None:
source = self.model.last_call
newnodes = m.dNodeList(self.model.parent_graph)
for x in self.model:
subimages = MakeSubimages(x[source], stride, size)
for im in subimages:
imnode = Node(self.model.parent_graph, {
'type': 'subimage',
'image': im
})
newnodes.append(imnode)
x.Connect(imnode)
return newnodes
def LoadCSV(filename: str, separator: str = ",") -> m.dNodeList:
'''loads csv into nodelist using CSV reader'''
import neuthink.metagraph as m
graph = Graph()
lines: m.dNodeList = m.dNodeList(graph)
f = open(filename)
csv_reader = csv.reader(f, delimiter=separator)
first_column = True
try:
for row in csv_reader:
if first_column:
column_names = row
# print(column_names)
first_column = False
else:
_node = {column_names[i]: row[i] for i in range(len(row))}
node = Node(graph, _node)
lines.append(node)
except:
print("error occured while parsing csv")
return lines
from neuthink import metagraph as m
#from neuthink.textviews import SentenceView
from neuthink.graph.basics import Graph
import torch
import os
graph = Graph()
path = os.path.dirname(os.path.abspath(__file__))
print(path)
model = m.dNodeList(graph)
model.classes = [
x.strip()
for x in open(path + '/' + 'model3000x2x20482.cls', 'r').readlines()
]
import json
model.resources = json.loads(
open(path + '/' + 'model3000x2x20482.res', 'r').read())
model.Text.VectorizeLookupChars(source="chunk",
target="chunk_tensor",
alphabet="alphabetchunk_tensor")
model.Lookup(source="chunk_tensor", target="Lookup1", size=50)
model.LSTM(size=3192,
source="Lookup1",
target="LSTM1",
input_size=50,
state_vector="default",
carry_state="no")
model.LSTM(size=2048,
source="LSTM1",
target="LSTM2",
input_size=3192,
state_vector="default",