def loader():
rankings_country_code_population = transform.transformer()
########################################################
# load data to postgres
########################################################
#create database connection
engine = create_engine(
f'postgresql://{config.postgres_conn_str}{config.upload_database}')
#insert df into postgres
rankings_country_code_population.to_sql(name='country_rankings',
con=engine,
if_exists='replace',
index=False)
########################################################
# load data to mongodb
########################################################
# Initialize PyMongo to work with MongoDBs
conn = 'mongodb://localhost:27017'
client = pymongo.MongoClient(conn)
# Define database and collection
db = client.etl_project
collection = db.country_rankings
doc = rankings_country_code_population.set_index('country').to_dict()
results = collection.find()
doc_id = {}
for i in results:
doc_id['_id'] = str(i.get('_id'))
collection.replace_one(doc_id, doc, upsert=True)
def proxy_query(self, flow, url, query):
parts = urlparse.urlparse(url)
# Get resource type - first try to see whether there is type= URL option,
# if there is not, try to get it from file extension
if parts.scheme != "http":
raise ResourceError("Only HTTP URLs are supported", "Data proxy does not support %s URLs" % parts.scheme)
resource_type = query.getfirst("type")
if not resource_type:
resource_type = os.path.splitext(parts.path)[1]
if not resource_type:
raise RequestError(
"Could not determine the resource type",
"If file has no type extension, specify file type in type= option",
)
resource_type = re.sub(r"^\.", "", resource_type.lower())
try:
transformer = transform.transformer(resource_type, flow, url, query)
except Exception, e:
raise RequestError(
"Resource type not supported",
"Transformation of resource of type %s is not supported. Reason: %s" % (resource_type, e),
)
def proxy_query(self, flow, url, query):
parts = urlparse.urlparse(url)
# Get resource type - first try to see whether there is type= URL option,
# if there is not, try to get it from file extension
if parts.scheme not in ['http', 'https']:
raise ResourceError(
'Only HTTP URLs are supported',
'Data proxy does not support %s URLs' % parts.scheme)
resource_type = query.getfirst("type")
if not resource_type:
resource_type = os.path.splitext(parts.path)[1]
if not resource_type:
raise RequestError(
'Could not determine the resource type',
'If file has no type extension, specify file type in type= option'
)
resource_type = re.sub(r'^\.', '', resource_type.lower())
try:
transformer = transform.transformer(resource_type, flow, url,
query)
except Exception, e:
raise RequestError(
'Resource type not supported',
'Transformation of resource of type %s is not supported. Reason: %s'
% (resource_type, e))
def transform_preprocessed_ndi_files(self,st):
bad_list = []
for name in self.matched_list:
pname = name[1]+"-preprocessed"
processed_ndi_file = os.path.join(self.processed,pname)
my_transform = tf.transformer(processed_ndi_file,st)
if (my_transform.process_file()):
my_transform.save_processed_file()
else:
bad_list.append(name)
for val in bad_list:
self.matched_list.remove(val)
src = os.path.join(self.archive_good,val[1])
shutil.move(src,self.archive_bad)
# clean up the failed ndi file and its matching pair from processed good to processed bad.
return
def compile_file(f, name, c):
base, ext = os.path.splitext(name)
print 'read...'
r = lisp_reader.reader(f)
exp = r.read_all()
if c.verbose:
print '--- read ---'
pp(exp)
print 'transform...'
t = transform.transformer(c)
exp2 = t.go(exp)
if c.verbose:
print '--- transform ---'
pp(exp2)
w = nodes.walker(c)
exp3 = w.go(exp2)
print 'rename...'
# alpha conversion
c.var_dict = nodes.rename_variables(exp3, c)
# find strongly connected components
print 'call graph...'
c.dep_graph = graph.build_dependency_graph(exp3)
c.scc_graph, c.scc_map = graph.strongly(c.dep_graph)
a = analyze.analyzer(c)
exp4 = a.analyze(exp3)
if c.verbose:
print '--- analyzer ---'
exp4.pprint()
ic = byte_cps(c, verbose=c.verbose)
exp5 = ic.go(exp4)
if c.verbose:
print '--- cps ---'
cps.pretty_print(exp5)
fo = open('%s.byc' % base, 'wb')
num_regs = cps.the_register_allocator.max_reg
b = compiler(fo, name, num_regs, c)
b.go(exp5)
fo.close()
def compile_file (f, name, c):
base, ext = os.path.splitext (name)
print 'read...'
r = lisp_reader.reader (f)
exp = r.read_all()
if c.verbose:
print '--- read ---'
pp (exp)
print 'transform...'
t = transform.transformer (c)
exp2 = t.go (exp)
if c.verbose:
print '--- transform ---'
pp (exp2)
w = nodes.walker (c)
exp3 = w.go (exp2)
print 'rename...'
# alpha conversion
c.var_dict = nodes.rename_variables (exp3, c)
# find strongly connected components
print 'call graph...'
c.dep_graph = graph.build_dependency_graph (exp3)
c.scc_graph, c.scc_map = graph.strongly (c.dep_graph)
a = analyze.analyzer (c)
exp4 = a.analyze (exp3)
if c.verbose:
print '--- analyzer ---'
exp4.pprint()
ic = byte_cps (c, verbose=c.verbose)
exp5 = ic.go (exp4)
if c.verbose:
print '--- cps ---'
cps.pretty_print (exp5)
fo = open ('%s.byc' % base, 'wb')
num_regs = cps.the_register_allocator.max_reg
b = compiler (fo, name, num_regs, c)
b.go (exp5)
fo.close()
b_fc_loc1 = bias_variable([20])
initial = np.array([0.5, 0])
initial = initial.astype('float32')
initial = initial.flatten()
W_fc_loc2 = weight_variable([20, 2])
b_fc_loc2 = bias_variable([2])
b_fc_loc2 = tf.Variable(initial_value=initial, name='b_fc_loc2')
h_fc_loc1 = tf.nn.tanh(tf.matmul(x_flat, W_fc_loc1) + b_fc_loc1)
h_fc_loc1_drop = tf.nn.dropout(h_fc_loc1, keep_prob)
h_fc_loc2 = tf.nn.tanh(tf.matmul(h_fc_loc1_drop, W_fc_loc2) + b_fc_loc2)
out_size = (10, 4096)
h_trans = transformer(x_trans, h_fc_loc2, out_size)
h_flat = tf.reshape(h_trans, [-1, 10, 4096, 1])
# start cnn
#filter_size = 3
filter_size = 5
n_filters_1 = 8
W_conv1 = weight_variable_cnn([filter_size, 1, 1, n_filters_1])
b_conv1 = bias_variable([n_filters_1])
h_conv1 = tf.nn.relu(
tf.nn.conv2d(
input=h_trans, filter=W_conv1, strides=[1, 1, 1, 1], padding='SAME') +
b_conv1)
h_conv1_flat = tf.reshape(h_conv1, [-1, 10 * 4096 * n_filters_1])
