Exemple #1
0
def loader():
    rankings_country_code_population = transform.transformer()

    ########################################################
    # load data to postgres
    ########################################################
    #create database connection
    engine = create_engine(
        f'postgresql://{config.postgres_conn_str}{config.upload_database}')
    #insert df into postgres
    rankings_country_code_population.to_sql(name='country_rankings',
                                            con=engine,
                                            if_exists='replace',
                                            index=False)

    ########################################################
    # load data to mongodb
    ########################################################
    # Initialize PyMongo to work with MongoDBs
    conn = 'mongodb://localhost:27017'
    client = pymongo.MongoClient(conn)
    # Define database and collection
    db = client.etl_project
    collection = db.country_rankings
    doc = rankings_country_code_population.set_index('country').to_dict()

    results = collection.find()
    doc_id = {}

    for i in results:
        doc_id['_id'] = str(i.get('_id'))

    collection.replace_one(doc_id, doc, upsert=True)
Exemple #2
0
    def proxy_query(self, flow, url, query):
        parts = urlparse.urlparse(url)

        # Get resource type - first try to see whether there is type= URL option,
        # if there is not, try to get it from file extension

        if parts.scheme != "http":
            raise ResourceError("Only HTTP URLs are supported", "Data proxy does not support %s URLs" % parts.scheme)

        resource_type = query.getfirst("type")
        if not resource_type:
            resource_type = os.path.splitext(parts.path)[1]

        if not resource_type:
            raise RequestError(
                "Could not determine the resource type",
                "If file has no type extension, specify file type in type= option",
            )

        resource_type = re.sub(r"^\.", "", resource_type.lower())

        try:
            transformer = transform.transformer(resource_type, flow, url, query)
        except Exception, e:
            raise RequestError(
                "Resource type not supported",
                "Transformation of resource of type %s is not supported. Reason: %s" % (resource_type, e),
            )
Exemple #3
0
    def proxy_query(self, flow, url, query):
        parts = urlparse.urlparse(url)

        # Get resource type - first try to see whether there is type= URL option,
        # if there is not, try to get it from file extension

        if parts.scheme not in ['http', 'https']:
            raise ResourceError(
                'Only HTTP URLs are supported',
                'Data proxy does not support %s URLs' % parts.scheme)

        resource_type = query.getfirst("type")
        if not resource_type:
            resource_type = os.path.splitext(parts.path)[1]

        if not resource_type:
            raise RequestError(
                'Could not determine the resource type',
                'If file has no type extension, specify file type in type= option'
            )

        resource_type = re.sub(r'^\.', '', resource_type.lower())

        try:
            transformer = transform.transformer(resource_type, flow, url,
                                                query)
        except Exception, e:
            raise RequestError(
                'Resource type not supported',
                'Transformation of resource of type %s is not supported. Reason: %s'
                % (resource_type, e))
Exemple #4
0
 def transform_preprocessed_ndi_files(self,st):
     bad_list = []
     for name in self.matched_list:
         pname = name[1]+"-preprocessed"
         processed_ndi_file = os.path.join(self.processed,pname)
         my_transform = tf.transformer(processed_ndi_file,st)
         if (my_transform.process_file()):
             my_transform.save_processed_file()
         else:
             bad_list.append(name)
     for val in bad_list:
         self.matched_list.remove(val)
         src = os.path.join(self.archive_good,val[1])
         shutil.move(src,self.archive_bad)
         # clean up the failed ndi file and its matching pair from processed good to processed bad.
     return
Exemple #5
0
def compile_file(f, name, c):
    base, ext = os.path.splitext(name)
    print 'read...'
    r = lisp_reader.reader(f)
    exp = r.read_all()

    if c.verbose:
        print '--- read ---'
        pp(exp)

    print 'transform...'
    t = transform.transformer(c)
    exp2 = t.go(exp)
    if c.verbose:
        print '--- transform ---'
        pp(exp2)

    w = nodes.walker(c)
    exp3 = w.go(exp2)

    print 'rename...'
    # alpha conversion
    c.var_dict = nodes.rename_variables(exp3, c)
    # find strongly connected components
    print 'call graph...'
    c.dep_graph = graph.build_dependency_graph(exp3)
    c.scc_graph, c.scc_map = graph.strongly(c.dep_graph)

    a = analyze.analyzer(c)
    exp4 = a.analyze(exp3)

    if c.verbose:
        print '--- analyzer ---'
        exp4.pprint()

    ic = byte_cps(c, verbose=c.verbose)
    exp5 = ic.go(exp4)

    if c.verbose:
        print '--- cps ---'
        cps.pretty_print(exp5)

    fo = open('%s.byc' % base, 'wb')
    num_regs = cps.the_register_allocator.max_reg
    b = compiler(fo, name, num_regs, c)
    b.go(exp5)
    fo.close()
def compile_file (f, name, c):
    base, ext = os.path.splitext (name)
    print 'read...'
    r = lisp_reader.reader (f)
    exp = r.read_all()

    if c.verbose:
        print '--- read ---'
        pp (exp)

    print 'transform...'
    t = transform.transformer (c)
    exp2 = t.go (exp)
    if c.verbose:
        print '--- transform ---'
        pp (exp2)

    w = nodes.walker (c)
    exp3 = w.go (exp2)

    print 'rename...'
    # alpha conversion
    c.var_dict = nodes.rename_variables (exp3, c)
    # find strongly connected components
    print 'call graph...'
    c.dep_graph = graph.build_dependency_graph (exp3)
    c.scc_graph, c.scc_map = graph.strongly (c.dep_graph)

    a = analyze.analyzer (c)
    exp4 = a.analyze (exp3)

    if c.verbose:
        print '--- analyzer ---'
        exp4.pprint()

    ic = byte_cps (c, verbose=c.verbose)
    exp5 = ic.go (exp4)

    if c.verbose:
        print '--- cps ---'
        cps.pretty_print (exp5)

    fo = open ('%s.byc' % base, 'wb')
    num_regs = cps.the_register_allocator.max_reg
    b = compiler (fo, name, num_regs, c)
    b.go (exp5)
    fo.close()
Exemple #7
0
b_fc_loc1 = bias_variable([20])

initial = np.array([0.5, 0])
initial = initial.astype('float32')
initial = initial.flatten()

W_fc_loc2 = weight_variable([20, 2])
b_fc_loc2 = bias_variable([2])
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')

h_fc_loc1 = tf.nn.tanh(tf.matmul(x_flat, W_fc_loc1) + b_fc_loc1)
h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob)
h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1_drop, W_fc_loc2) + b_fc_loc2)

out_size = (10, 4096)
h_trans = transformer(x_trans, h_fc_loc2, out_size)
h_flat = tf.reshape(h_trans, [-1, 10, 4096, 1])

# start cnn

#filter_size = 3
filter_size = 5
n_filters_1 = 8
W_conv1 = weight_variable_cnn([filter_size, 1, 1, n_filters_1])
b_conv1 = bias_variable([n_filters_1])
h_conv1 = tf.nn.relu(
    tf.nn.conv2d(
        input=h_trans, filter=W_conv1, strides=[1, 1, 1, 1], padding='SAME') +
    b_conv1)

h_conv1_flat = tf.reshape(h_conv1, [-1, 10 * 4096 * n_filters_1])