def main():
db = Database(DATABASE)
db.update()
count = 1
total = len(db.tables)
for table in db.tables:
url = db.tables[table]['url']
timezone = db.tables[table]['timezone']
try:
site = BeautifulSoup(urlopen(url))
except urllib.error.HTTPError:
print("error on %s" % (url))
data = []
for tr in site('tr'):
raw_datum = tr.text.strip().split('\n')
try:
date = datetime.strptime(
raw_datum[0],
"%d/%m/%Y %H:%M").replace(tzinfo=pytz.timezone(timezone))
value = float(raw_datum[1])
datum = (date, value)
except ValueError:
continue
data.append(datum)
db.store_data(data, table)
sleep(5)
print("%d / %d is complete" % (count, total))
count += 1
def main():
db = Database(DATABASE)
db.update()
count = 1
total = len(db.tables)
for table in db.tables:
url = db.tables[table]['url']
timezone = db.tables[table]['timezone']
try:
site = BeautifulSoup(urlopen(url))
except urllib.error.HTTPError:
print ("error on %s" %(url))
data = []
for tr in site('tr'):
raw_datum = tr.text.strip().split('\n')
try:
date = datetime.strptime(raw_datum[0], "%d/%m/%Y %H:%M").replace(
tzinfo=pytz.timezone(timezone)
)
value = float(raw_datum[1])
datum = (date, value)
except ValueError:
continue
data.append(datum)
db.store_data(data, table)
sleep(5)
print ("%d / %d is complete" %(count, total))
count += 1
def _get_labeled(self, pseudo_count):
db = Database(fullpath("data/data.db"))
data = Data(mediatype="movie")
titles = data.titles_list()
labeled = []
try:
select_existing = db.select(table="main", fields="all")
except OperationalError:
data.collect(args=data.titles_list(),
addtl_items=["boxoffice", "oscars", "review"])
i = 0
for movie in labeled:
if i == 0:
print("Making Table main")
db.make_table(table="main",
fields=[k for k in movie.keys()])
else:
pass
db.update(row=movie)
db.conn.commit()
i += 1
else:
if len(select_existing) == len(titles):
header = select_existing[0].keys()
for x in select_existing:
labeled.append(dict(map(lambda x, y: (x, y), header, x)))
else:
data.collect(args=data.titles_list(),
addtl_items=["boxoffice", "oscars", "review"])
i = 0
for movie in labeled:
if i == 0:
print("Making Table main")
db.make_table(table="main",
fields=[k for k in movie.keys()])
else:
pass
db.update(row=movie)
db.conn.commit()
i += 1
lab = self._pseudo_cases(data=labeled, n=pseudo_count, classx="0")
return lab
