def show(): # names = ['realtimehot', 'relevant_topic', 'users', 'words'] # for name in names: # data = select(name) # print(data) # for key in data.keys(): # dateArray = datetime.datetime.utcfromtimestamp(key) # otherStyleTime = dateArray.strftime("%Y-%m-%d %H:%M:%S") # print(otherStyleTime + ":" + str(data[key])) # data = select("topic") data = select("realtimehot") time = [] numbers = {} for key in data.keys(): dateArray = datetime.datetime.utcfromtimestamp(key) otherStyleTime = dateArray.strftime("%Y-%m-%d %H:%M:%S") time.append(otherStyleTime) for item in data[key].keys(): # numbers[item] = [] try: numbers[item][0].append(otherStyleTime) numbers[item][1].append(data[key][item]) except KeyError: numbers[item] = [[], []] numbers[item][0].append(otherStyleTime) numbers[item][1].append(data[key][item]) # print(item) # print(data[key][item]) print(time) print(numbers) matplotlib.pyplot.title = "Realtimehot" matplotlib.pyplot.xlabel = "Time" matplotlib.pyplot.ylabel = "Clicks" # matplotlib.pyplot.rcParams['figure.dpi'] = 300 # matplotlib.pyplot.rcParams['figure.figsize'] = (8.0, 4.0) for key in numbers.keys(): print(numbers[key][0]) print(numbers[key][1]) matplotlib.pyplot.plot(numbers[key][0], numbers[key][1], label=key) # break matplotlib.pyplot.show()
def main(): print("[Date Format : YYYYMMDD]") start = input("Start Date : ") end = input("End Date : ") start = datetime.datetime(int(start[:4]), int(start[4:6]), int(start[6:]), 1, 1) end = datetime.datetime(int(end[:4]), int(end[4:6]), int(end[6:]), 23, 59) conn = connect() # db connection res = select(conn) # get raw data table = [['Datetime', 'IP', 'CPU', 'RAM']] for row in res: date = row[0].split('T')[0].split('-') date = datetime.datetime(int(date[0]), int(date[1]), int(date[2]), 11, 11) if date > start and date < end: table.append(row) with open('data.csv', 'w', newline='') as csvfile: # export as data.csv writer = csv.writer(csvfile) writer.writerows(table) conn.close() # close connection
def save_and_check(self, dataIn, tableIn, dataOut, tableOut = None, twice = True): if tableOut == None: tableOut = quote(tableIn) # Insert sqlite.save([], dataIn, tableIn) # Observe with pysqlite connection=sqlite3.connect(self.DBNAME) cursor=connection.cursor() cursor.execute("SELECT * FROM %s" % tableOut) observed1 = cursor.fetchall() connection.close() if twice: # Observe with DumpTruck observed2 = sqlite.select('* FROM %s' % tableOut) #Check expected1 = dataOut expected2 = [dataIn] if type(dataIn) == dict else dataIn self.assertListEqual(observed1, expected1) self.assertListEqual(observed2, expected2)
def test_select(self): shutil.copy('dumptruck/fixtures/landbank_branches.sqlite',self.DBNAME) sqlite._connect(self.DBNAME) data_observed = sqlite.select("* FROM `branches` WHERE Fax is not null ORDER BY Fax LIMIT 3;") data_expected = [{'town': u'\r\nCenturion', 'date_scraped': 1327791915.618461, 'Fax': u' (012) 312 3647', 'Tel': u' (012) 686 0500', 'address_raw': u'\r\n420 Witch Hazel Ave\n\r\nEcopark\n\r\nCenturion\n\r\n0001\n (012) 686 0500\n (012) 312 3647', 'blockId': 14, 'street-address': None, 'postcode': u'\r\n0001', 'address': u'\r\n420 Witch Hazel Ave\n\r\nEcopark\n\r\nCenturion\n\r\n0001', 'branchName': u'Head Office'}, {'town': u'\r\nCenturion', 'date_scraped': 1327792245.787187, 'Fax': u' (012) 312 3647', 'Tel': u' (012) 686 0500', 'address_raw': u'\r\n420 Witch Hazel Ave\n\r\nEcopark\n\r\nCenturion\n\r\n0001\n (012) 686 0500\n (012) 312 3647', 'blockId': 14, 'street-address': u'\r\n420 Witch Hazel Ave\n\r\nEcopark', 'postcode': u'\r\n0001', 'address': u'\r\n420 Witch Hazel Ave\n\r\nEcopark\n\r\nCenturion\n\r\n0001', 'branchName': u'Head Office'}, {'town': u'\r\nMiddelburg', 'date_scraped': 1327791915.618461, 'Fax': u' (013) 282 6558', 'Tel': u' (013) 283 3500', 'address_raw': u'\r\n184 Jan van Riebeeck Street\n\r\nMiddelburg\n\r\n1050\n (013) 283 3500\n (013) 282 6558', 'blockId': 17, 'street-address': None, 'postcode': u'\r\n1050', 'address': u'\r\n184 Jan van Riebeeck Street\n\r\nMiddelburg\n\r\n1050', 'branchName': u'Middelburg'}] self.assertListEqual(data_observed, data_expected)
def hsph(self): #https://scraperwiki.com/scrapers/export_sqlite/hsph_faculty_1/ sqlite.attach('hsph_faculty_1') observed = sqlite.select('count(*) as "c" from maincol')[0]['c'] expected = 461 self.assertEqual(observed, expected)
def save_and_select(self, d): sqlite.save([], {"foo": d}) observed = sqlite.select('* from swdata')[0]['foo'] self.assertEqual(d, observed)