def Compile(obj):
if isinstance(obj,str):
assert obj.endswith('.json'),'Config type not currently supported'
obj = json.load(open(obj,'r'))
layers = []
COMPONENTS = {}
if 'IMPORTS' in obj:
assert isinstance(obj['IMPORTS'],list), 'BLOX IMPORTS must be a string'
for f in obj['IMPORTS']:
if isinstance(f,str):
if f.endswith('.blx') or f.endswith('.json'):
_obj = json.load( open(f,'r') )
obj_name = _obj['Name'] if 'Name' in _obj else ''.join(f.split('.')[:-1])
COMPONENTS.update(
{
obj_name:Compile( _obj )
}
)
else:
COMPONENTS.update( load( f ) )
# USER_DEFINED.update( load( f ) )
if 'DEFS' in obj:
for k,fs in obj['DEFS']['BLOX'].items():COMPONENTS.update( {k: nn.Sequential(*[handlers[k](v) for f in fs for k,v in f.items() ]) } )
# for k,fs in obj['DEFS'].items():COMPONENTS.update( {k: nn.Sequential(*[handlers[k](v) for f in fs for k,v in f.items() ]) } )
if 'BLOX' in obj:
for layer in obj['BLOX']:
if isinstance(layer,str):
f = layer
print(layer,json.dumps(list(PREDEFINED.keys()),indent=2))
if f in COMPONENTS:
funcs = COMPONENTS[f]
elif f in PREDEFINED:
funcs = PREDEFINED[f]()
else:
raise ValueError('Function Block not defined in config file. Error @ {}'.format(f))
layers.append( funcs )
else:
if 'MODULE' in layer:
layers.append(PREDEFINED[layer['MODULE']['LAYER']](**layer['MODULE']['ARGS']) if isinstance(layer['MODULE'],dict) else PREDEFINED[layer['MODULE']]() )
elif 'DEF' in layer:
f = layer['DEF']
# if the block is defined in another file,load and continue
if f.endswith('.json') or f.endswith('.blx'):
funcs = Compile( json.load( open(f,'r') ) )
# check to see if the block is previously defined
elif f in COMPONENTS:
funcs = COMPONENTS[f]
elif f in PREDEFINED:
funcs = PREDEFINED[f]()
else:
raise ValueError('Function Block not defined in config file. Error @ {}'.format(f))
layers.append( funcs )
elif 'REPEAT' in layer:
b = layer['REPEAT']['BLOCK']
t = layer['REPEAT']['REPS']
layers.append( nn.Sequential(*[ c for _ in range(t) for c in COMPONENTS[b] ]) if isinstance(COMPONENTS[b],list) else COMPONENTS[b] )
else:
for k,v in layer.items():
layers.append( handlers[k](v) )
return nn.Sequential(*layers).cuda() if torch.cuda.is_available() else nn.Sequential(*layers)
def __init__(self,args):
try: self.config = json.load(open(args.config,'r'))
except:
if isinstance(args,dict): self.config = args
elif isinstance(args,str): self.config = json.load(open(args,'r'))
else:raise ValueError('Incorrect data type passed to Trainer class')
if self.config['Verbose']:print(TEXT+('\n'*8))
def __call__(self, adjacency, image_list, labels, transform):
with open(adjacency, 'r') as J:
adjacency = yajl.load(J)
with open(image_list, 'r') as J:
image_list = yajl.load(J)['image_list']
transforms = {'sketch_transform': sketch_transform}
transform = transforms.get(transform, sketch_transform)()
return self.base_module(adjacency, image_list, labels, transform)
def FetchConfig(j = 'dev'):
'''Fetching configuration from local JSON file.'''
j = j + '.json'
data_path = os.path.split(dir)[0]
try:
j = os.path.join(data_path, 'config', j)
with open(j) as json_file:
data = json.load(json_file)
except Exception as e:
print "Could not load configuration."
print e
return False
#
# Perform basic quality checks.
#
if len(data['hdx_key']) is not 36:
print '%s API key seems to be wrong. Please check: %s' % (I('prompt_error'), os.path.split(j)[1])
return False
return data
def StrOrDict(cfg):
if isinstance(cfg, str):
if cfg.endswith('.json') or cfg.endswith('.cfg'):
cfg = json.load(open(cfg, 'r'))
else:
cfg = json.loads(cfg)
return cfg
def read_cache(cache_file):
logger.debug("Reading cache: {}".format(cache_file))
try:
with codecs.open(cache_file, 'r', 'utf-8') as f:
cache = json.load(f)
# If the file is not found Python 3.4 will raise FileNotFoundError which is
# a subclass of IOError.
# See also:
# http://sebastianraschka.com/Articles/python3_OSError.html
except IOError:
cache = dict()
return cache
def load(source):
"""
Use json.load for back-ward compatibility, since cjson doesn't
provide this method. The load method works on file-descriptor
objects.
"""
if MODULE == 'json':
return json.load(source)
elif MODULE == 'cjson':
data = source.read()
return cjson.decode(data)
elif MODULE == 'yajl':
return yajl.load(source)
else:
raise Exception("Not support JSON module: %s" % MODULE)
def load(source):
"""
Use json.load for back-ward compatibility, since cjson doesn't
provide this method. The load method works on file-descriptor
objects.
"""
if MODULE == 'json':
return json.load(source)
elif MODULE == 'cjson':
data = source.read()
return cjson.decode(data)
elif MODULE == 'yajl':
return yajl.load(source)
else:
raise Exception("Not support JSON module: %s" % MODULE)
def LoadData(p, verbose=True):
'''Loading data from a local JSON resource.'''
if verbose:
print "--------------------------------------------------"
print "%s Loading JSON data from %s." % (I('bullet'), p)
try:
data = json.load(open(p))
if verbose:
print "%s Data loaded successully. %s entities in dataset." % (
I('bullet'), len(data))
print "--------------------------------------------------"
except Exception as e:
print "%s Could not load %s file." % (I('error'), p)
return None
#
# Return data.
#
return data
def LoadData(p, verbose=True):
'''Loading data from a local JSON resource.'''
if verbose:
print "--------------------------------------------------"
print "%s Loading JSON data from %s." % (I('prompt_bullet').decode('utf-8'), p)
try:
data = json.load(open(p))
if verbose:
print "%s Data loaded successully. %s entities in dataset." % (I('prompt_bullet').decode('utf-8'), len(data[0]))
print "--------------------------------------------------"
except Exception as e:
print "%s Could not load %s file." % (I('prompt_error'), p)
return None
#
# Perform basic quality checks.
#
return data[0]
def LoadData(p, verbose=True):
'''Loading data from a local JSON resource.'''
if verbose:
print("--------------------------------------------------")
print("%s Loading JSON data from %s." % (I('bullet'), p))
try:
data = json.load(open(p))
if verbose:
print("%s Data loaded successully. %s entities in dataset." % (I('bullet'), len(data)))
print("--------------------------------------------------")
except Exception as e:
print("%s Could not load %s file." % (I('error'), p))
return None
#
# Return data.
#
return data
def Run(self, cmd_val):
arg_r = args.Reader(cmd_val.argv, spids=cmd_val.arg_spids)
arg_r.Next() # skip 'json'
action, action_spid = arg_r.Peek2()
if action is None:
raise error.Usage(_JSON_ACTION_ERROR)
arg_r.Next()
if action == 'write':
arg, _ = JSON_WRITE_SPEC.Parse(arg_r)
# GetVar() of each name and print it.
for var_name in arg_r.Rest():
if var_name.startswith(':'):
var_name = var_name[1:]
val = self.mem.GetVar(var_name)
with tagswitch(val) as case:
if case(value_e.Undef):
# TODO: blame the right span_id
self.errfmt.Print("no variable named %r is defined",
var_name)
return 1
elif case(value_e.Str):
obj = val.s
elif case(value_e.MaybeStrArray):
obj = val.strs
elif case(value_e.AssocArray):
obj = val.d
elif case(value_e.Obj):
obj = val.obj
else:
raise AssertionError(val)
if arg.pretty:
indent = arg.indent
extra_newline = False
else:
# How yajl works: if indent is -1, then everything is on one line.
indent = -1
extra_newline = True
j = yajl.dump(obj, sys.stdout, indent=indent)
if extra_newline:
sys.stdout.write('\n')
# TODO: Accept a block. They aren't hooked up yet.
if cmd_val.block:
# TODO: flatten value.{Str,Obj} into a flat dict?
namespace = self.cmd_ev.EvalBlock(cmd_val.block)
print(yajl.dump(namespace))
elif action == 'read':
arg, _ = JSON_READ_SPEC.Parse(arg_r)
# TODO:
# Respect -validate=F
var_name, name_spid = arg_r.ReadRequired2("expected variable name")
if var_name.startswith(':'):
var_name = var_name[1:]
if not match.IsValidVarName(var_name):
raise error.Usage('got invalid variable name %r' % var_name,
span_id=name_spid)
try:
# Use a global _STDIN, because we get EBADF on a redirect if we use a
# local. A Py_DECREF closes the file, which we don't want, because the
# redirect is responsible for freeing it.
#
# https://github.com/oilshell/oil/issues/675
#
# TODO: write a better binding like yajl.readfd()
#
# It should use streaming like here:
# https://lloyd.github.io/yajl/
obj = yajl.load(_STDIN)
except ValueError as e:
self.errfmt.Print('json read: %s', e, span_id=action_spid)
return 1
self.mem.SetVar(sh_lhs_expr.Name(var_name), value.Obj(obj),
scope_e.LocalOnly)
else:
raise error.Usage(_JSON_ACTION_ERROR, span_id=action_spid)
return 0
def load_options(options_file) :
with open(options_file, 'r') as J :
options = yajl.load(J)
options = Namespace(**options)
return options
def test_simple_decode(self):
obj = yajl.load(self.stream)
self.assertEquals(obj, {'foo' : ['one', 'two', ['three', 'four']]})
parser = ArgumentParser()
parser.add_argument('--ip',
default='0.0.0.0',
help='ip address (default: 0.0.0.0)',
action='store')
parser.add_argument('--port',
default=8080,
help='port (default: 8080)',
action='store')
parser.add_argument('--tag', default='', action='store')
parser.add_argument('--config', action='store_true')
args = parser.parse_args()
app = web.Application()
app.add_routes(routes)
if args.tag:
tag = args.tag
if args.config:
if os.path.isfile('config.json'):
cfg = 'config.json'
else:
cfg = 'config.example.json'
with open(cfg) as f:
tag = load(f)['tag']
web.run_app(app, host=args.ip, port=args.port)
def __call__(self, cmd_val):
arg_r = args.Reader(cmd_val.argv, spids=cmd_val.arg_spids)
arg_r.Next() # skip 'json'
action, action_spid = arg_r.Peek2()
if action is None:
raise args.UsageError(_JSON_ACTION_ERROR)
arg_r.Next()
if action == 'write':
arg, _ = JSON_WRITE_SPEC.Parse(arg_r)
# GetVar() of each name and print it.
for var_name in arg_r.Rest():
if var_name.startswith(':'):
var_name = var_name[1:]
val = self.mem.GetVar(var_name)
with tagswitch(val) as case:
if case(value_e.Undef):
# TODO: blame the right span_id
self.errfmt.Print("no variable named %r is defined",
var_name)
return 1
elif case(value_e.Str):
obj = val.s
elif case(value_e.MaybeStrArray):
obj = val.strs
elif case(value_e.AssocArray):
obj = val.d
elif case(value_e.Obj):
obj = val.obj
else:
raise AssertionError(val)
if arg.pretty:
indent = arg.indent
extra_newline = False
else:
# How yajl works: if indent is -1, then everything is on one line.
indent = -1
extra_newline = True
j = yajl.dump(obj, sys.stdout, indent=indent)
if extra_newline:
sys.stdout.write('\n')
# TODO: Accept a block. They aren't hooked up yet.
if cmd_val.block:
# TODO: flatten value.{Str,Obj} into a flat dict?
namespace = self.ex.EvalBlock(cmd_val.block)
print(yajl.dump(namespace))
elif action == 'read':
arg, _ = JSON_READ_SPEC.Parse(arg_r)
# TODO:
# Respect -validate=F
var_name, name_spid = arg_r.ReadRequired2("expected variable name")
if var_name.startswith(':'):
var_name = var_name[1:]
if not match.IsValidVarName(var_name):
raise args.UsageError('got invalid variable name %r' %
var_name,
span_id=name_spid)
# Have to use this over sys.stdin because of redirects
# TODO: change binding to yajl.readfd() ?
stdin = posix_.fdopen(0)
try:
obj = yajl.load(stdin)
except ValueError as e:
self.errfmt.Print('json read: %s', e, span_id=action_spid)
return 1
self.mem.SetVar(sh_lhs_expr.Name(var_name), value.Obj(obj), (),
scope_e.LocalOnly)
else:
raise args.UsageError(_JSON_ACTION_ERROR, span_id=action_spid)
return 0
def test_simple_decode(self):
obj = yajl.load(self.stream)
self.assertEquals(obj, {'foo' : ['one', 'two', ['three', 'four']]})
from graph_algo import edge_to_adjacency
# With transforms
# -----------------------------------
from torchvision.transforms import Compose, Grayscale, ToTensor
from torchvision.transforms import Resize, RandomCrop
T = Compose([Grayscale(), Resize(224), RandomCrop(224), ToTensor()])
## Json Loader
# -----------------------------------
import yajl
combinations_json = '/home/bvr/data/pytosine/k_nearest/20180526-153919/combinations.json'
with open(combinations_json, 'r') as J:
similar_pairs = yajl.load(J)['combinations']
lg.info('Loaded similar pairs: size:%s', len(similar_pairs))
adjacency = edge_to_adjacency(similar_pairs)
# TODO: include edge_to_adjacency before tangle
image_list_json = '/home/bvr/data/pytosine/k_nearest/20180521-205730/image_list.json'
image_list_key = 'image_list'
with open(image_list_json, 'r') as J:
image_list = yajl.load(J)[image_list_key]
lg.info('Loaded image_list: size:%s', len(image_list))
def test_dataset(dataset_name):
global adjacency, image_list, T
dataset = dataset_name(adjacency,
import yajl,sys,os
import glob
import shapely.geometry
from shapely.ops import cascaded_union
for input in glob.glob("by_state/*.json"):
base = os.path.basename(input).replace(".json","")
print base
d = yajl.load(open(input))
polys = [shapely.geometry.shape(f['geometry']) for f in d['features'] ]
union = cascaded_union(polys)
print union.area
with open("unions/" + base+".wkb", 'w') as out:
out.write(union.wkb)
rawList[i][j] = re.sub('\[.*\]','',rawList[i][j])
rawList[i][j] = rawList[i][j].replace(':','')
# try:
rawList[i][j] = rawList[i][j].encode('ascii','ignore')
rawList[i][j] = (rawList[i][j]).strip()
# except:
# pass
return rawList
def buildDirectories(dirPath="wikipages"):
if not os.path.exists(dirPath):
os.makedirs(dirPath)
if os.path.isfile("links.json"):
with open("links.json") as json_file:
links = json.load(json_file)
buildDirectories("wikipages")
buildDirectories("wikipages/movies")
init_globals()
#PAGES = 100
#PAGES = 10
PAGES = 15
#for i in range(40,PAGES):
#for i in range(PAGES):
for i in range(10,PAGES):
# year = "19"+str(i)
# year = "200"+str(i)
