Ejemplo n.º 1
0
def Compile(obj):
    if isinstance(obj,str):
        assert obj.endswith('.json'),'Config type not currently supported'
        obj = json.load(open(obj,'r'))
    layers = []
    COMPONENTS = {}
    if 'IMPORTS' in obj:
        assert isinstance(obj['IMPORTS'],list), 'BLOX IMPORTS must be a string'
        for f in obj['IMPORTS']:
            if isinstance(f,str):
                if f.endswith('.blx') or f.endswith('.json'):
                    _obj = json.load( open(f,'r') ) 
                    obj_name = _obj['Name'] if 'Name' in _obj else  ''.join(f.split('.')[:-1])
                    COMPONENTS.update( 
                            { 
                                obj_name:Compile( _obj ) 
                            } 
                        )
                else: 
                    COMPONENTS.update( load( f )  )
                    # USER_DEFINED.update( load( f )  )
    if 'DEFS' in obj:
        for k,fs in obj['DEFS']['BLOX'].items():COMPONENTS.update( {k: nn.Sequential(*[handlers[k](v) for f in fs for k,v in f.items() ]) } )
        # for k,fs in obj['DEFS'].items():COMPONENTS.update( {k: nn.Sequential(*[handlers[k](v) for f in fs for k,v in f.items() ]) } )
    if 'BLOX' in obj:
        for layer in obj['BLOX']:
            if isinstance(layer,str):                
                f = layer
                print(layer,json.dumps(list(PREDEFINED.keys()),indent=2))
                if f in COMPONENTS:
                    funcs = COMPONENTS[f]
                elif f in PREDEFINED:
                    funcs = PREDEFINED[f]()
                else:
                    raise ValueError('Function Block not defined in config file. Error @ {}'.format(f))
                layers.append( funcs )
            else:
                if 'MODULE' in layer:
                    layers.append(PREDEFINED[layer['MODULE']['LAYER']](**layer['MODULE']['ARGS']) if isinstance(layer['MODULE'],dict) else PREDEFINED[layer['MODULE']]() )
                elif 'DEF' in layer:
                    f = layer['DEF']
                    # if the block is defined in another file,load and continue
                    if f.endswith('.json') or f.endswith('.blx'):
                        funcs = Compile( json.load( open(f,'r') ) )
                    # check to see if the block is previously defined
                    elif f in COMPONENTS:
                        funcs = COMPONENTS[f]
                    elif f in PREDEFINED:
                        funcs = PREDEFINED[f]()
                    else:
                        raise ValueError('Function Block not defined in config file. Error @ {}'.format(f))
                    layers.append( funcs )
                elif 'REPEAT' in layer:
                    b = layer['REPEAT']['BLOCK']
                    t = layer['REPEAT']['REPS']
                    layers.append( nn.Sequential(*[ c for _ in range(t) for c in COMPONENTS[b] ]) if isinstance(COMPONENTS[b],list) else COMPONENTS[b] ) 
                else:
                    for k,v in layer.items():
                        layers.append( handlers[k](v) )
    return nn.Sequential(*layers).cuda() if torch.cuda.is_available() else nn.Sequential(*layers)
Ejemplo n.º 2
0
    def __init__(self,args):
        try: self.config = json.load(open(args.config,'r'))
        except: 
            if isinstance(args,dict): self.config = args
            elif isinstance(args,str): self.config = json.load(open(args,'r'))
            else:raise ValueError('Incorrect data type passed to Trainer class')

        if self.config['Verbose']:print(TEXT+('\n'*8))
Ejemplo n.º 3
0
    def __call__(self, adjacency, image_list, labels, transform):
        with open(adjacency, 'r') as J:
            adjacency = yajl.load(J)

        with open(image_list, 'r') as J:
            image_list = yajl.load(J)['image_list']

        transforms = {'sketch_transform': sketch_transform}
        transform = transforms.get(transform, sketch_transform)()

        return self.base_module(adjacency, image_list, labels, transform)
Ejemplo n.º 4
0
def FetchConfig(j = 'dev'):
  '''Fetching configuration from local JSON file.'''

  j = j + '.json'

  data_path = os.path.split(dir)[0]

  try:
    j = os.path.join(data_path, 'config', j)
    with open(j) as json_file:    
      data = json.load(json_file)

  except Exception as e:
    print "Could not load configuration."
    print e
    return False

  #
  # Perform basic quality checks.
  #
  if len(data['hdx_key']) is not 36:
    print '%s API key seems to be wrong. Please check: %s' % (I('prompt_error'), os.path.split(j)[1])
    return False

  return data
Ejemplo n.º 5
0
def StrOrDict(cfg):
    if isinstance(cfg, str):
        if cfg.endswith('.json') or cfg.endswith('.cfg'):
            cfg = json.load(open(cfg, 'r'))
        else:
            cfg = json.loads(cfg)
    return cfg
Ejemplo n.º 6
0
def read_cache(cache_file):
    logger.debug("Reading cache: {}".format(cache_file))
    try:
        with codecs.open(cache_file, 'r', 'utf-8') as f:
            cache = json.load(f)

    # If the file is not found  Python 3.4 will raise FileNotFoundError which is
    # a subclass of IOError.
    # See also:
    # http://sebastianraschka.com/Articles/python3_OSError.html
    except IOError:
        cache = dict()

    return cache
Ejemplo n.º 7
0
def load(source):
    """
    Use json.load for back-ward compatibility, since cjson doesn't
    provide this method. The load method works on file-descriptor
    objects.
    """
    if  MODULE == 'json':
        return json.load(source)
    elif MODULE == 'cjson':
        data = source.read()
        return cjson.decode(data)
    elif MODULE == 'yajl':
        return yajl.load(source)
    else:
        raise Exception("Not support JSON module: %s" % MODULE)
Ejemplo n.º 8
0
def load(source):
    """
    Use json.load for back-ward compatibility, since cjson doesn't
    provide this method. The load method works on file-descriptor
    objects.
    """
    if MODULE == 'json':
        return json.load(source)
    elif MODULE == 'cjson':
        data = source.read()
        return cjson.decode(data)
    elif MODULE == 'yajl':
        return yajl.load(source)
    else:
        raise Exception("Not support JSON module: %s" % MODULE)
Ejemplo n.º 9
0
def LoadData(p, verbose=True):
    '''Loading data from a local JSON resource.'''

    if verbose:
        print "--------------------------------------------------"
        print "%s Loading JSON data from %s." % (I('bullet'), p)

    try:
        data = json.load(open(p))

        if verbose:
            print "%s Data loaded successully. %s entities in dataset." % (
                I('bullet'), len(data))
            print "--------------------------------------------------"

    except Exception as e:
        print "%s Could not load %s file." % (I('error'), p)
        return None

    #
    # Return data.
    #
    return data
Ejemplo n.º 10
0
def LoadData(p, verbose=True):
  '''Loading data from a local JSON resource.'''
  
  if verbose:
    print "--------------------------------------------------"
    print "%s Loading JSON data from %s." % (I('prompt_bullet').decode('utf-8'), p)

  try:
      data = json.load(open(p))

      if verbose:
          print "%s Data loaded successully. %s entities in dataset." % (I('prompt_bullet').decode('utf-8'), len(data[0]))
          print "--------------------------------------------------"


  except Exception as e:
      print "%s Could not load %s file." % (I('prompt_error'), p)
      return None


  #
  # Perform basic quality checks.
  #
  return data[0]
Ejemplo n.º 11
0
def LoadData(p, verbose=True):
  '''Loading data from a local JSON resource.'''

  if verbose:
    print("--------------------------------------------------")
    print("%s Loading JSON data from %s." % (I('bullet'), p))

  try:
      data = json.load(open(p))

      if verbose:
          print("%s Data loaded successully. %s entities in dataset." % (I('bullet'), len(data)))
          print("--------------------------------------------------")


  except Exception as e:
      print("%s Could not load %s file." % (I('error'), p))
      return None


  #
  # Return data.
  #
  return data
Ejemplo n.º 12
0
    def Run(self, cmd_val):
        arg_r = args.Reader(cmd_val.argv, spids=cmd_val.arg_spids)
        arg_r.Next()  # skip 'json'

        action, action_spid = arg_r.Peek2()
        if action is None:
            raise error.Usage(_JSON_ACTION_ERROR)
        arg_r.Next()

        if action == 'write':
            arg, _ = JSON_WRITE_SPEC.Parse(arg_r)

            # GetVar() of each name and print it.

            for var_name in arg_r.Rest():
                if var_name.startswith(':'):
                    var_name = var_name[1:]

                val = self.mem.GetVar(var_name)
                with tagswitch(val) as case:
                    if case(value_e.Undef):
                        # TODO: blame the right span_id
                        self.errfmt.Print("no variable named %r is defined",
                                          var_name)
                        return 1
                    elif case(value_e.Str):
                        obj = val.s
                    elif case(value_e.MaybeStrArray):
                        obj = val.strs
                    elif case(value_e.AssocArray):
                        obj = val.d
                    elif case(value_e.Obj):
                        obj = val.obj
                    else:
                        raise AssertionError(val)

                if arg.pretty:
                    indent = arg.indent
                    extra_newline = False
                else:
                    # How yajl works: if indent is -1, then everything is on one line.
                    indent = -1
                    extra_newline = True

                j = yajl.dump(obj, sys.stdout, indent=indent)
                if extra_newline:
                    sys.stdout.write('\n')

            # TODO: Accept a block.  They aren't hooked up yet.
            if cmd_val.block:
                # TODO: flatten value.{Str,Obj} into a flat dict?
                namespace = self.cmd_ev.EvalBlock(cmd_val.block)

                print(yajl.dump(namespace))

        elif action == 'read':
            arg, _ = JSON_READ_SPEC.Parse(arg_r)
            # TODO:
            # Respect -validate=F

            var_name, name_spid = arg_r.ReadRequired2("expected variable name")
            if var_name.startswith(':'):
                var_name = var_name[1:]

            if not match.IsValidVarName(var_name):
                raise error.Usage('got invalid variable name %r' % var_name,
                                  span_id=name_spid)

            try:
                # Use a global _STDIN, because we get EBADF on a redirect if we use a
                # local.  A Py_DECREF closes the file, which we don't want, because the
                # redirect is responsible for freeing it.
                #
                # https://github.com/oilshell/oil/issues/675
                #
                # TODO: write a better binding like yajl.readfd()
                #
                # It should use streaming like here:
                # https://lloyd.github.io/yajl/

                obj = yajl.load(_STDIN)
            except ValueError as e:
                self.errfmt.Print('json read: %s', e, span_id=action_spid)
                return 1

            self.mem.SetVar(sh_lhs_expr.Name(var_name), value.Obj(obj),
                            scope_e.LocalOnly)

        else:
            raise error.Usage(_JSON_ACTION_ERROR, span_id=action_spid)

        return 0
Ejemplo n.º 13
0
def load_options(options_file) :
  with open(options_file, 'r') as J :
    options = yajl.load(J)

  options = Namespace(**options)
  return options
Ejemplo n.º 14
0
 def test_simple_decode(self):
     obj = yajl.load(self.stream)
     self.assertEquals(obj, {'foo' : ['one', 'two', ['three', 'four']]})
Ejemplo n.º 15
0
    parser = ArgumentParser()
    parser.add_argument('--ip',
                        default='0.0.0.0',
                        help='ip address (default: 0.0.0.0)',
                        action='store')
    parser.add_argument('--port',
                        default=8080,
                        help='port (default: 8080)',
                        action='store')
    parser.add_argument('--tag', default='', action='store')
    parser.add_argument('--config', action='store_true')

    args = parser.parse_args()

    app = web.Application()
    app.add_routes(routes)

    if args.tag:
        tag = args.tag

    if args.config:
        if os.path.isfile('config.json'):
            cfg = 'config.json'
        else:
            cfg = 'config.example.json'
        with open(cfg) as f:
            tag = load(f)['tag']

    web.run_app(app, host=args.ip, port=args.port)
Ejemplo n.º 16
0
    def __call__(self, cmd_val):
        arg_r = args.Reader(cmd_val.argv, spids=cmd_val.arg_spids)
        arg_r.Next()  # skip 'json'

        action, action_spid = arg_r.Peek2()
        if action is None:
            raise args.UsageError(_JSON_ACTION_ERROR)
        arg_r.Next()

        if action == 'write':
            arg, _ = JSON_WRITE_SPEC.Parse(arg_r)

            # GetVar() of each name and print it.

            for var_name in arg_r.Rest():
                if var_name.startswith(':'):
                    var_name = var_name[1:]

                val = self.mem.GetVar(var_name)
                with tagswitch(val) as case:
                    if case(value_e.Undef):
                        # TODO: blame the right span_id
                        self.errfmt.Print("no variable named %r is defined",
                                          var_name)
                        return 1
                    elif case(value_e.Str):
                        obj = val.s
                    elif case(value_e.MaybeStrArray):
                        obj = val.strs
                    elif case(value_e.AssocArray):
                        obj = val.d
                    elif case(value_e.Obj):
                        obj = val.obj
                    else:
                        raise AssertionError(val)

                if arg.pretty:
                    indent = arg.indent
                    extra_newline = False
                else:
                    # How yajl works: if indent is -1, then everything is on one line.
                    indent = -1
                    extra_newline = True

                j = yajl.dump(obj, sys.stdout, indent=indent)
                if extra_newline:
                    sys.stdout.write('\n')

            # TODO: Accept a block.  They aren't hooked up yet.
            if cmd_val.block:
                # TODO: flatten value.{Str,Obj} into a flat dict?
                namespace = self.ex.EvalBlock(cmd_val.block)

                print(yajl.dump(namespace))

        elif action == 'read':
            arg, _ = JSON_READ_SPEC.Parse(arg_r)
            # TODO:
            # Respect -validate=F

            var_name, name_spid = arg_r.ReadRequired2("expected variable name")
            if var_name.startswith(':'):
                var_name = var_name[1:]

            if not match.IsValidVarName(var_name):
                raise args.UsageError('got invalid variable name %r' %
                                      var_name,
                                      span_id=name_spid)

            # Have to use this over sys.stdin because of redirects
            # TODO: change binding to yajl.readfd() ?
            stdin = posix_.fdopen(0)
            try:
                obj = yajl.load(stdin)
            except ValueError as e:
                self.errfmt.Print('json read: %s', e, span_id=action_spid)
                return 1

            self.mem.SetVar(sh_lhs_expr.Name(var_name), value.Obj(obj), (),
                            scope_e.LocalOnly)

        else:
            raise args.UsageError(_JSON_ACTION_ERROR, span_id=action_spid)

        return 0
Ejemplo n.º 17
0
 def test_simple_decode(self):
     obj = yajl.load(self.stream)
     self.assertEquals(obj, {'foo' : ['one', 'two', ['three', 'four']]})
Ejemplo n.º 18
0
    from graph_algo import edge_to_adjacency

    # With transforms
    # -----------------------------------
    from torchvision.transforms import Compose, Grayscale, ToTensor
    from torchvision.transforms import Resize, RandomCrop
    T = Compose([Grayscale(), Resize(224), RandomCrop(224), ToTensor()])

    ## Json Loader
    # -----------------------------------
    import yajl

    combinations_json = '/home/bvr/data/pytosine/k_nearest/20180526-153919/combinations.json'
    with open(combinations_json, 'r') as J:
        similar_pairs = yajl.load(J)['combinations']
    lg.info('Loaded similar pairs: size:%s', len(similar_pairs))

    adjacency = edge_to_adjacency(similar_pairs)
    # TODO: include edge_to_adjacency before tangle

    image_list_json = '/home/bvr/data/pytosine/k_nearest/20180521-205730/image_list.json'
    image_list_key = 'image_list'
    with open(image_list_json, 'r') as J:
        image_list = yajl.load(J)[image_list_key]
    lg.info('Loaded image_list: size:%s', len(image_list))

    def test_dataset(dataset_name):
        global adjacency, image_list, T

        dataset = dataset_name(adjacency,
Ejemplo n.º 19
0
import yajl,sys,os
import glob
import shapely.geometry
from shapely.ops import cascaded_union

for input in glob.glob("by_state/*.json"):
  base = os.path.basename(input).replace(".json","")
  print base

  d = yajl.load(open(input))
  polys = [shapely.geometry.shape(f['geometry']) for f in d['features'] ]
  union = cascaded_union(polys)
  print union.area
  with open("unions/" + base+".wkb", 'w') as out:
    out.write(union.wkb)


Ejemplo n.º 20
0
                rawList[i][j] = re.sub('\[.*\]','',rawList[i][j])
                rawList[i][j] = rawList[i][j].replace(':','')
#                try:
                rawList[i][j] = rawList[i][j].encode('ascii','ignore')
                rawList[i][j] = (rawList[i][j]).strip()
#                except:
#                    pass
    return rawList

def buildDirectories(dirPath="wikipages"):
    if not os.path.exists(dirPath):
        os.makedirs(dirPath)

if os.path.isfile("links.json"):
    with open("links.json") as json_file:
        links = json.load(json_file)


buildDirectories("wikipages")
buildDirectories("wikipages/movies")
init_globals()
#PAGES = 100
#PAGES = 10
PAGES = 15

#for i in range(40,PAGES):
#for i in range(PAGES):
for i in range(10,PAGES):

#    year = "19"+str(i)
#    year = "200"+str(i)