def show():
# names = ['realtimehot', 'relevant_topic', 'users', 'words']
# for name in names:
# data = select(name)
# print(data)
# for key in data.keys():
# dateArray = datetime.datetime.utcfromtimestamp(key)
# otherStyleTime = dateArray.strftime("%Y-%m-%d %H:%M:%S")
# print(otherStyleTime + ":" + str(data[key]))
# data = select("topic")
data = select("realtimehot")
time = []
numbers = {}
for key in data.keys():
dateArray = datetime.datetime.utcfromtimestamp(key)
otherStyleTime = dateArray.strftime("%Y-%m-%d %H:%M:%S")
time.append(otherStyleTime)
for item in data[key].keys():
# numbers[item] = []
try:
numbers[item][0].append(otherStyleTime)
numbers[item][1].append(data[key][item])
except KeyError:
numbers[item] = [[], []]
numbers[item][0].append(otherStyleTime)
numbers[item][1].append(data[key][item])
# print(item)
# print(data[key][item])
print(time)
print(numbers)
matplotlib.pyplot.title = "Realtimehot"
matplotlib.pyplot.xlabel = "Time"
matplotlib.pyplot.ylabel = "Clicks"
# matplotlib.pyplot.rcParams['figure.dpi'] = 300
# matplotlib.pyplot.rcParams['figure.figsize'] = (8.0, 4.0)
for key in numbers.keys():
print(numbers[key][0])
print(numbers[key][1])
matplotlib.pyplot.plot(numbers[key][0], numbers[key][1], label=key)
# break
matplotlib.pyplot.show()
def main():
print("[Date Format : YYYYMMDD]")
start = input("Start Date : ")
end = input("End Date : ")
start = datetime.datetime(int(start[:4]), int(start[4:6]), int(start[6:]), 1, 1)
end = datetime.datetime(int(end[:4]), int(end[4:6]), int(end[6:]), 23, 59)
conn = connect() # db connection
res = select(conn) # get raw data
table = [['Datetime', 'IP', 'CPU', 'RAM']]
for row in res:
date = row[0].split('T')[0].split('-')
date = datetime.datetime(int(date[0]), int(date[1]), int(date[2]), 11, 11)
if date > start and date < end:
table.append(row)
with open('data.csv', 'w', newline='') as csvfile: # export as data.csv
writer = csv.writer(csvfile)
writer.writerows(table)
conn.close() # close connection
def save_and_check(self, dataIn, tableIn, dataOut, tableOut = None, twice = True):
if tableOut == None:
tableOut = quote(tableIn)
# Insert
sqlite.save([], dataIn, tableIn)
# Observe with pysqlite
connection=sqlite3.connect(self.DBNAME)
cursor=connection.cursor()
cursor.execute("SELECT * FROM %s" % tableOut)
observed1 = cursor.fetchall()
connection.close()
if twice:
# Observe with DumpTruck
observed2 = sqlite.select('* FROM %s' % tableOut)
#Check
expected1 = dataOut
expected2 = [dataIn] if type(dataIn) == dict else dataIn
self.assertListEqual(observed1, expected1)
self.assertListEqual(observed2, expected2)
def test_select(self):
shutil.copy('dumptruck/fixtures/landbank_branches.sqlite',self.DBNAME)
sqlite._connect(self.DBNAME)
data_observed = sqlite.select("* FROM `branches` WHERE Fax is not null ORDER BY Fax LIMIT 3;")
data_expected = [{'town': u'\r\nCenturion', 'date_scraped': 1327791915.618461, 'Fax': u' (012) 312 3647', 'Tel': u' (012) 686 0500', 'address_raw': u'\r\n420 Witch Hazel Ave\n\r\nEcopark\n\r\nCenturion\n\r\n0001\n (012) 686 0500\n (012) 312 3647', 'blockId': 14, 'street-address': None, 'postcode': u'\r\n0001', 'address': u'\r\n420 Witch Hazel Ave\n\r\nEcopark\n\r\nCenturion\n\r\n0001', 'branchName': u'Head Office'}, {'town': u'\r\nCenturion', 'date_scraped': 1327792245.787187, 'Fax': u' (012) 312 3647', 'Tel': u' (012) 686 0500', 'address_raw': u'\r\n420 Witch Hazel Ave\n\r\nEcopark\n\r\nCenturion\n\r\n0001\n (012) 686 0500\n (012) 312 3647', 'blockId': 14, 'street-address': u'\r\n420 Witch Hazel Ave\n\r\nEcopark', 'postcode': u'\r\n0001', 'address': u'\r\n420 Witch Hazel Ave\n\r\nEcopark\n\r\nCenturion\n\r\n0001', 'branchName': u'Head Office'}, {'town': u'\r\nMiddelburg', 'date_scraped': 1327791915.618461, 'Fax': u' (013) 282 6558', 'Tel': u' (013) 283 3500', 'address_raw': u'\r\n184 Jan van Riebeeck Street\n\r\nMiddelburg\n\r\n1050\n (013) 283 3500\n (013) 282 6558', 'blockId': 17, 'street-address': None, 'postcode': u'\r\n1050', 'address': u'\r\n184 Jan van Riebeeck Street\n\r\nMiddelburg\n\r\n1050', 'branchName': u'Middelburg'}]
self.assertListEqual(data_observed, data_expected)
def hsph(self):
#https://scraperwiki.com/scrapers/export_sqlite/hsph_faculty_1/
sqlite.attach('hsph_faculty_1')
observed = sqlite.select('count(*) as "c" from maincol')[0]['c']
expected = 461
self.assertEqual(observed, expected)
def save_and_select(self, d):
sqlite.save([], {"foo": d})
observed = sqlite.select('* from swdata')[0]['foo']
self.assertEqual(d, observed)
