def parse_faults(self):
print ('Parsing shapefiles for %s' % self.shapefile)
layer = fiona.open(self.shapefile)
# layer.schema
data = []
for f in layer:
d = dict(f['properties'])
name = d.get('name', None)
ftype = d.get('ftype', None)
length = d.get('length', None)
sliprate = d.get('sliprate', None)
slipcode = d.get('slipcode', None)
slipsense = d.get('slipsense', None)
age = d.get('age', None)
if f['geometry']:
line_str = self.line_str(f['geometry']['coordinates'][0])
else:
line_str = None
data.append((name, ftype, length, sliprate, slipcode, slipsense,
age, line_str, None))
query = "INSERT INTO faults VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)"
dbo.execute_query(self.conn, query, data, multiple=True)
return
def load_shapefile(self, filepath):
# state = filepath.split('/')[-2]
print('Parsing shapefiles for %s' % filepath)
if self.check_file(filepath):
return
layer = fiona.open(filepath)
# layer.schema
data = []
for f in layer:
try:
mtrs = f['properties']['mtrs']
except:
try:
mtrs = f['properties']['MTRS']
except:
mtrs = None
claim_id = f['id']
poly_str = self.polygon_str(f['geometry']['coordinates'][0])
if len(poly_str) > 1000:
continue
data.append((claim_id, mtrs, poly_str, None))
query = "INSERT INTO claims_geo VALUES (%s, %s, %s, %s)"
dbo.execute_query(self.conn, query, data, multiple=True)
return
def insert_miner_news(self, article):
data = [(article["link"], article["title"], article["ticker"],
article["source"], article["desc"], article["date"])]
try:
query = "INSERT INTO company_news VALUES (%s, %s, %s, %s, %s, %s)"
except:
print("Issue inserting data:")
print(data)
dbo.execute_query(self.conn, query, data, multiple=True)
return
def _storeData(self, data, table, query=None):
"""Interal function to store scraped data"""
print('Storing data')
conn = dbo.getConnection()
if query == None:
num_cols = len(data[0])
cols = ','.join(['%s ' for i in range(0, num_cols)])
query = "INSERT INTO " + table + " VALUES (" + cols + ")"
dbo.execute_query(conn, query, data, multiple=True)
dbo.closeConnection(conn)
return
def load_claims_meta(self, filepath):
print('Parsing text file %s' % filepath)
if self.check_file(filepath):
return
f = open(filepath, 'r')
data = []
for i, line in enumerate(f.readlines()):
if i == 0:
continue
line = line.strip().split(',')
data.append(tuple(line))
query = "INSERT INTO claims_meta VALUES (%s, %s, %s, %s)"
dbo.execute_query(self.conn, query, data, multiple=True)
return
def _updateScraped(self, table, record_type):
"""Internal function to set scraped=True in the records tracking
table
"""
print('Updating scraped ids from %s' % table)
conn = dbo.getConnection()
query = """
UPDATE records
SET scraped=True
FROM %s AS updater
WHERE updater.id=records.id
AND records.type=%s
""" % (table, "'{0}'".format(record_type))
dbo.execute_query(conn, query)
dbo.closeConnection(conn)
return
import src.database_operations as dbo
conn = dbo.db_connect()
f = open('data/mines.txt', encoding='latin-1')
lines = f.readlines()
data = [tuple(line.strip().split('\t')) for line in lines]
# for x in data:
# if len(x[2])>200:
# print (len(x[2]), x[2])
try:
query = "INSERT INTO public.mines VALUES (%s, %s, %s, %s, %s, %s, %s)"
dbo.execute_query(conn, query, data[1:], multiple=True)
except Exception as e:
print(e)
conn.close()
def historize_results(self, prop, results):
data = [(prop['mine_id'], prop['mine_name'], r['link'], r['title'],
r['citedby'], r['author']) for r in results]
query = "INSERT INTO google_scholar VALUES (%s, %s, %s, %s, %s, %s)"
dbo.execute_query(self.conn, query, data, multiple=True)
return
def insert_miner(self, miner):
data = [(miner["name"], miner["url"], miner["ticker"],
miner["market_cap"])]
query = "INSERT INTO companies VALUES (%s, %s, %s, %s)"
dbo.execute_query(self.conn, query, data, multiple=True)
return
def historize_results(self, prop, results):
data = [(prop['mine_id'], prop['mine_name'], r['link'],
r['title'], r['desc'], r['source'], r['date']) for r in results]
query = "INSERT INTO google_news VALUES (%s, %s, %s, %s, %s, %s, %s)"
dbo.execute_query(self.conn, query, data, multiple=True)
return