def run(self):
(options, args) = self.parser.parse_args()
self.options = options
self.args = args
runner1, runner2, profile1, profile2 = self.get_runners()
if sys.platform == 'linux2':
try:
from Xlib import X, display
d = display.Display()
s = d.screen()
root = s.root
root.warp_pointer(0, 0)
d.sync()
except ImportError:
print "Xlib is not installed. Mouse may interfere with screenshots."
if options.test is True:
all_sites = test_all_sites
store = False
else:
all_sites = None
store = True
if options.report is None:
db = couchquery.Database("http://127.0.0.1:5984/sitecompare")
else:
if '@' in options.report:
user, password = options.report.strip('http://').split(
'@')[0].split(':')
url = 'http://' + options.report.split('@')[1]
http = httplib2.Http('.cache')
http.add_credentials(user, password)
db = couchquery.Database(url, http=http)
else:
db = couchquery.Database(options.report)
c = CompareSites(runner1,
runner2,
options.report,
self.run_info,
all_sites=all_sites,
store=store,
db=db)
c.start()
sleep(2)
c.stop()
sleep(3)
def setup():
from brasstacks import fennec
olddb = couchquery.Database('http://localhost:5984/fennec_old')
db = couchquery.Database('http://localhost:5984/fennec')
db.sync_design_doc('fennec', fennec.design_doc)
db.sync_design_doc('fennecFailures', fennec.failures_design_doc)
a = Stub()
a.add_resource('fennec', fennec.FennecApplication(db))
global thread
global httpd
httpd = make_server('', 8888, a)
thread = Thread(target=httpd.serve_forever)
thread.start()
sleep(1)
return olddb, db
def sync_cli(db=None):
db = [i for i in sys.argv if i.startswith('http')]
assert len(db) is 1
dbname = db[0]
db = couchquery.Database(dbname, cache=Cache())
sys.argv.remove(dbname)
sync(db, sys.argv)
def setup_module(module):
# from brasstacks import crashtest
crashdb = couchquery.Database(crashtestcouch + '/crashtest')
resultdb = couchquery.Database(crashtestcouch + '/crashtest_results')
# crashdb.sync_design_doc("crashes", crashtest.crashes_design_doc)
# resultdb.sync_design_doc("jobs", crashtest.jobs_design_doc)
# resultdb.sync_design_doc("results", crashtest.results_design_doc)
# a = Stub()
# a.add_resource('crashtest', crashtest.CrashTestApplication(crashdb, resultdb))
# httpd = make_server('', 8888, a)
# thread = Thread(target=httpd.serve_forever)
# thread.start()
#
# # Global these boys
# module.thread = thread
# module.httpd = httpd
module.crashdb = crashdb
module.resultdb = resultdb
def main():
document = 'sample_doc_for_testing_and_dev'
products = ['Fennec', 'Firefox']
platforms = ['Maemo-n810', 'WinMo'] #, 'Linux', 'Mac']
testtypes = ['crashtests', 'mochitests'] #, 'xpcshell', 'reftests']
build = timestamp = ''
# db = couchquery.Database("http://pythonesque.org:5984/logcompare", cache=Cache())
db = couchquery.Database(
"http://*****:*****@happyhans.couch.io/logcompare",
cache=Cache())
# create documents
doccount = random.randint(40, 70)
for i in range(0, doccount):
# create metadata
buildstructure = {}
buildstructure['build'] = random.randint(999000, 999999)
buildstructure['product'] = random.choice(products)
buildstructure['os'] = random.choice(platforms)
buildstructure['testtype'] = random.choice(testtypes)
buildstructure['timestamp'] = str(datetime.datetime.now())
buildstructure['document'] = document
buildstructure['tinderboxID'] = random.choice(
[-1, str(datetime.datetime.now())])
# create tests
tests = {}
offset = random.randrange(1, 5)
testcount = random.randint(100, 500)
for x in range(0, testcount):
failcount = random.randint(0, 10)
todocount = random.randint(0, 3)
notes = []
for y in range(0, (failcount + todocount)):
# notes.append("This test should have returned TRUE but returned FALSE")
notes.append("Message!")
tests['test_' + str(offset + x) + '.js'] = {
'pass': random.randint(0, 5),
'fail': failcount,
'todo': todocount,
'note': notes
}
print json.dumps(buildstructure, indent=2)
buildstructure['tests'] = tests
# outputFile = "C:/_Code/python/outputdata" + str(i) + ".html"
# outFile = open(outputFile, 'w')
# outFile.write(json.dumps(buildstructure, indent=2))
# outFile.close()
db.create(buildstructure)
print "done uploading results"
def __init__(self, uri, flag='c', writeback=False, raiseconflicts=False, http_cache=None):
self._db = couchquery.Database(uri, cache=http_cache)
if flag == 'n':
couchquery.deletedb(self._db)
if flag in ('n', 'c'):
if not self._db.exists():
couchquery.createdb(self._db)
self._flag = flag
self._writeback = writeback
self._raiseconflicts = raiseconflicts
self._cache = {}
def cli():
db = [i for i in sys.argv if i.startswith('http')]
assert len(db) is 1
dbname = db[0]
db = couchquery.Database(dbname, cache=Cache())
sys.argv.remove(dbname)
sync(db, sys.argv)
a = get_application(db, sys.argv)
httpd = make_server('', 8888, a)
print "Serving on http://localhost:8888/"
httpd.serve_forever()
def test_simple_set_get(self):
d = shelve.open(URI)
d['item1'] = []
d.close()
d = shelve.open(URI)
self.assertEqual(d['item1'], [])
d.close()
# Make sure the data actually ended up in the database
db = couchquery.Database(URI)
doc = db.get('item1')
value = pickle.loads(str(doc.value))
self.assertEqual(value, [])
def test_confict_behavior(self):
# First prove that we will silently override conflicts
d = shelve.open(URI)
d['item1'] = []
db = couchquery.Database(URI)
doc = db.get('item1')
doc.value = pickle.dumps("NotAList")
db.save(doc)
d['item1'] = [1]
self.assertEqual(d['item1'], [1])
d.close()
def save(data):
#TODO: make the db a global variable or parameter to function
# db = couchquery.Database("http://pythonesque.org:5984/fennec_test", cache=Cache())
# db = couchquery.Database("http://*****:*****@happyhans.couch.io/logcompare", cache=Cache())
db = couchquery.Database("http://10.2.76.100:5984/fennec_alpha",
cache=Cache())
saved = False
try:
starttime = datetime.datetime.now()
print db.create(data)
finishtime = datetime.datetime.now()
print finishtime - starttime
saved = True
except CouchDBException, e:
print "Error occurred while sending data :" + str(e)
def get_application(db, names):
a = Stub()
a.add_resource('static', FileServerApplication(static_dir))
if 'sitecompare' in names:
from brasstacks.sitecompare import SiteCompareApplication
a.add_resource('sitecompare', SiteCompareApplication(db))
if 'logcompare' in names:
from brasstacks.logcompare import LogCompareApplication
logcompare_application = LogCompareApplication(db)
a.add_resource('logcompare', logcompare_application)
if 'users' in names or 'brasstacks' in names:
from brasstacks.users import UsersApplication
users_application = UsersApplication(db)
a.add_resource('users', users_application)
if 'tcm' in names:
from brasstacks.tcm import TestCaseManagerApplication
tcm_application = TestCaseManagerApplication(db)
a.add_resource('tcm', tcm_application)
if 'fennec' in names:
from brasstacks.fennec import FennecApplication
fennec_application = FennecApplication(db)
a.add_resource('fennec', fennec_application)
if 'mozmill' in names:
from brasstacks.mozmill import MozmillApplication
mozmill_application = MozmillApplication(db)
a.add_resource('mozmill', mozmill_application)
if 'crashtest' in names:
resultdb = couchquery.Database(db.uri[:-1] + '_results')
a.add_resource('crashtest',
crashtest.CrashTestApplication(db, resultdb))
if 'firefox' in names:
from brasstacks.firefox import FirefoxApplication
firefox_application = FirefoxApplication(db)
a.add_resource('firefox', firefox_application)
return a
def main():
print "going"
#define your couchdb here
TARGET_HTTP = '115.146.86.188'
update_HTTP = '127.0.0.1'
TARGET_DB = 'melbourne_suburb'
weather_DB = 'weatherdb'
updatedb = 'coastshop'
COUCHDB_LINK = 'http://' + TARGET_HTTP + ':5984/' + TARGET_DB
weather_LINK = 'http://' + TARGET_HTTP + ':5984/' + weather_DB
updatedb_LINK = 'http://' + update_HTTP + ':5984/'
#################################################################################
# # to query the view for cities and weather
db = couchquery.Database(COUCHDB_LINK)
db_weather = couchquery.Database(weather_LINK)
# db_update = couchquery.Database(updatedb_LINK)
#################################################################################
# # to update summary to the new db
server = couchdb.Server(updatedb_LINK)
db_update = server[updatedb]
#################################################################################
# # necessary action to create 2D dictionary
def tree():
return collections.defaultdict(tree)
#################################################################################
# # sum each day each city total twitter number
rows = db.views.mapreduce.coastshop(group_level=2)
NumPerday = tree()
pnnDistri = tree()
for row in rows.items():
if (row[0][1] == 'coast'):
NumPerday[row[0][0]]['coast'] = row[1][0]
elif (row[0][1] == 'shop'):
NumPerday[row[0][0]]['shop'] = row[1][0]
else:
NumPerday[row[0][0]]['other'] = row[1][0]
#################################################################################
# # calculate the percentage of positive negative netrual for each day each city
rows = db.views.mapreduce.coastshop(group_level=3)
for row in rows.items():
if row[0][0] in NumPerday:
pnnDistri[row[0][0]][row[0][1]]['tweets'] = NumPerday[row[0][0]][
row[0][1]]
if row[0][2] == 1:
# pnnDistri[(row[0][0],1)] = [float(row[1][0])/NumPerday[row[0][0]]]
pnnDistri[row[0][0]][row[0][1]][1] = float(
row[1][0]) / NumPerday[row[0][0]][row[0][1]]
# print pnnDistri[(row[0][0],1)]
elif row[0][2] == 0:
# pnnDistri[(row[0][0],0)] = [float(row[1][0])/NumPerday[row[0][0]]]
pnnDistri[row[0][0]][row[0][1]][0] = float(
row[1][0]) / NumPerday[row[0][0]][row[0][1]]
# print pnnDistri[(row[0][0],0)]
else:
# pnnDistri[(row[0][0],-1)] = [float(row[1][0])/NumPerday[row[0][0]]]
pnnDistri[row[0][0]][row[0][1]][-1] = float(
row[1][0]) / NumPerday[row[0][0]][row[0][1]]
# print pnnDistri[(row[0][0],-1)]
# print pnnDistri
#################################################################################
# # withdraw weather from weather db and integrate with the emtion percentage
dateSuburb = tree()
rows = db_weather.views.mapreduce.Melbourne()
for row in rows.items():
if row[0][1] in pnnDistri:
for key, value in pnnDistri[row[0][1]].items():
smallkey = []
x = {}
x['region'] = key
x['date'] = row[0][1]
x['condition'] = row[0][2]
x['high'] = row[0][3]
x['low'] = row[0][4]
# if pnnDistri[row[0][1]][1] is not None:
x['positive'] = pnnDistri[row[0][1]][key][1]
# if pnnDistri[row[0][1]][1] is not None:
x['netrual'] = pnnDistri[row[0][1]][key][0]
# if pnnDistri[row[0][1]][1] is not None:
x['negative'] = pnnDistri[row[0][1]][key][-1]
# if pnnDistri[row[0][1]][1] is not None:
x['tweets'] = pnnDistri[row[0][1]][key]['tweets']
smallkey = (row[0][1], key)
dateSuburb[smallkey] = x
# print x
# print x
#################################################################################
# # convert to list to rearange the order
diclist = []
for key, value in dateSuburb.iteritems():
temp = [key, value]
diclist.append(temp)
# for key in diclist:
# print key[0][0],time.strptime(key[0][0],"%d %b %Y")
# timeobject = time.strptime(key[0][0],"%d %b %Y")
# print (datetime.datetime(timeobject.tm_year,timeobject.tm_mon,timeobject.tm_mday)-datetime.datetime(1970,1,1)).total_seconds()
# for key in diclist:
# print key[0]
diclist.sort(key=lambda date: time.strptime(date[0][0], "%d %b %Y"))
# for key in diclist:
# print key
# #################################################################################
# # # update to database
if len(db_update) > 0:
daterow = []
for id in db_update:
doc = db_update[id]
if ('date' in doc.keys()):
#################################################################################
# # use this tuple to index and indentify same day same city
daterow.append([(doc['date'], doc['region']), id])
# for key,value in daterow:
# print key,value
# print daterow
# for id in db_update:
# doc = db_update[id]
# if('date' in doc.keys()):
newIndex = [item[0] for item in daterow]
for key, value in diclist:
# print key,value
if (key in newIndex):
getid = [item[1] for item in daterow if item[0] == key]
print getid, type(getid[0])
newid = getid[0]
doc = db_update[newid]
if (value['positive'] != doc['positive']
or value['negative'] != doc['negative']
or value['netrual'] != doc['netrual']
or value['tweets'] != doc['tweets']):
doc['positive'] = value['positive']
doc['negative'] = value['negative']
doc['netrual'] = value['netrual']
db_update[newid] = doc
if (key not in newIndex):
print key
db_update.save(value)
else:
for key, value in diclist:
db_update.save(value)
def writeTempTweetcsv():
TARGET_HTTP = '115.146.86.188'
TARGET_DB = 'citysummary'
COUCHDB_LINK = 'http://' + TARGET_HTTP + ':5984/' + TARGET_DB
#################################################################################
# # to query the view for cities and weather
db = couchquery.Database(COUCHDB_LINK)
#################################################################################
# # calculate the percentage of positive negative netrual for each day each city
citynamelist = [
'Adelaide', 'Alice Springs', 'Brisbane', 'Canberra', 'Darwin',
'Hobart', 'Melbourne', 'Perth', 'Sydney'
]
for citynameitem in citynamelist:
rows = db.views.mapreduce.condition_deal(key=citynameitem)
rows_other = db.views.mapreduce.condition_deal(key="Adelaide")
dataframelist = []
dataframelist_other = []
#################################################################################
# # convert date to day month year
for row, other in zip(rows, rows_other):
# row[0] = time.strptime(row[0], "%d %m %Y %H %M %S")
row[0] = datetime.datetime.strptime(row[0], "%d %b %Y")
row[4] = float(row[4])
row[5] = float(row[5])
other[0] = datetime.datetime.strptime(other[0], "%d %b %Y")
other[4] = float(other[4])
other[5] = float(other[5])
dataframelist.append(row)
dataframelist_other.append(other)
##############################################################################
# withdraw corresponding data
df = pd.DataFrame(dataframelist,
columns=[
'date', 'positive', 'netrual', 'negative',
'high', 'low', 'diff', 'tweets', 'condition'
])
indexeddf = df.set_index(['date'])
df_other = pd.DataFrame(dataframelist_other,
columns=[
'date', 'positive', 'netrual', 'negative',
'high', 'low', 'diff', 'tweets',
'condition'
])
indexeddf_other = df_other.set_index(['date'])
newone = indexeddf.fillna(0)
##############################################################################
# draw each evariable pair cluster
twitterinfo = ['positive', 'netrual', 'negative', 'tweets']
weahterinfo = ['high', 'low', 'diff', 'condition']
for axisx in twitterinfo:
for axisy in weahterinfo:
namelist = axisx + "_" + axisy + "_" + citynameitem
print axisx, axisy
drawwDBSCAN(newone[[axisx, axisy]].values, namelist)
def writeTempTweetcsv():
TARGET_HTTP = '115.146.86.188'
TARGET_DB = 'citysummary'
COUCHDB_LINK = 'http://' + TARGET_HTTP + ':5984/' + TARGET_DB
#################################################################################
# # to query the view for cities and weather
db = couchquery.Database(COUCHDB_LINK)
#################################################################################
# # calculate the percentage of positive negative netrual for each day each city
citynamelist = [
'Adelaide', 'Alice Springs', 'Brisbane', 'Canberra', 'Darwin',
'Hobart', 'Melbourne', 'Perth', 'Sydney'
]
for citynameitem in citynamelist:
rows = db.views.mapreduce.condition_deal(key=citynameitem)
rows_other = db.views.mapreduce.condition_deal(key="Adelaide")
dataframelist = []
dataframelist_other = []
#################################################################################
# # convert date to day month year
for row, other in zip(rows, rows_other):
# row[0] = time.strptime(row[0], "%d %m %Y %H %M %S")
row[0] = datetime.datetime.strptime(row[0], "%d %b %Y")
row[4] = float(row[4])
row[5] = float(row[5])
other[0] = datetime.datetime.strptime(other[0], "%d %b %Y")
other[4] = float(other[4])
other[5] = float(other[5])
dataframelist.append(row)
dataframelist_other.append(other)
df = pd.DataFrame(dataframelist,
columns=[
'date', 'positive', 'netrual', 'negative',
'high', 'low', 'diff', 'tweets', 'condition'
])
indexeddf = df.set_index(['date'])
# drawCLuster(df[['positive','temperature']].values,"coast")
# drawGMM(df[['positive','temperature']].values,"coast")
df_other = pd.DataFrame(dataframelist_other,
columns=[
'date', 'positive', 'netrual', 'negative',
'high', 'low', 'diff', 'tweets',
'condition'
])
indexeddf_other = df_other.set_index(['date'])
dflookup_coast = pd.DataFrame(indexeddf,
columns=[
'positive', 'netrual', 'negative',
'high', 'low', 'diff', 'tweets',
'condition'
])
newone = indexeddf.fillna(0)
twitterinfo = ['positive', 'netrual', 'negative', 'tweets']
weahterinfo = ['high', 'low', 'diff', 'condition']
##############################################################################
# set parameters for DBSCAN
n_neighbors = 5
n_components = 2
namelist = "hour_dimention_weahterinfo_" + citynameitem
##############################################################################
# draw cluster according to Y and compareY after mds
Y = manifold.Isomap(n_neighbors, n_components).fit_transform(
newone[weahterinfo].values)
compareY = manifold.Isomap(n_neighbors, n_components).fit_transform(
newone[twitterinfo].values)
drawwDBSCAN(Y, compareY, namelist)
def tearDown(self):
# Make sure we leave with a clean-slate
db = couchquery.Database(URI)
response = db.http.get('')
if response.status == 200:
couchquery.deletedb(db)
def setUp(self):
# Make sure we start with a clean-slate
db = couchquery.Database(URI)
response = db.http.get('')
if response.status == 200:
couchquery.deletedb(db)
def writeTempTweetcsv():
TARGET_HTTP = '115.146.86.188'
TARGET_DB = 'coastshop_hour'
COUCHDB_LINK = 'http://' + TARGET_HTTP + ':5984/' + TARGET_DB
#################################################################################
# # to query the view for cities and weather
db = couchquery.Database(COUCHDB_LINK)
#################################################################################
# # calculate the percentage of positive negative netrual for each day each city
regionlist = ['coast', 'shop', 'other']
for region in regionlist:
rows = db.views.mapreduce.total_condi(key=region)
rows_other = db.views.mapreduce.total_condi(key="other")
dataframelist = []
dataframelist_other = []
#################################################################################
# # convert date to day month year hour minute second
for row, other in zip(rows, rows_other):
# row[0] = time.strptime(row[0], "%d %m %Y %H %M %S")
row[0] = datetime.datetime.strptime(row[0], "%d %m %Y %H %M %S")
row[6] = float(row[6])
row[7] = float(row[7])
row[8] = float(row[8])
row[9] = float(row[9])
other[0] = datetime.datetime.strptime(other[0],
"%d %m %Y %H %M %S")
other[6] = float(other[6])
other[7] = float(other[7])
other[8] = float(other[8])
other[9] = float(other[9])
dataframelist.append(row)
dataframelist_other.append(other)
##############################################################################
# withdraw corresponding data
df = pd.DataFrame(dataframelist,
columns=[
'date', 'positive', 'netrual', 'negative',
'tweets', 'condition', 'humidity', 'pressure',
'temperature', 'windSpeed'
])
indexeddf = df.set_index(['date'])
df_other = pd.DataFrame(dataframelist_other,
columns=[
'date', 'positive', 'netrual', 'negative',
'tweets', 'condition', 'humidity',
'pressure', 'temperature', 'windSpeed'
])
indexeddf_other = df_other.set_index(['date'])
##############################################################################
# draw each evariable pair cluster
twitterinfo = ['positive', 'netrual', 'negative', 'tweets']
weahterinfo = [
'condition', 'humidity', 'pressure', 'temperature', 'windSpeed'
]
for axisx in twitterinfo:
for axisy in weahterinfo:
namelist = axisx + "_" + axisy + "_" + region
drawwDBSCAN(indexeddf[[axisx, axisy]].values, namelist)
def writeTempTweetcsv():
TARGET_HTTP = '115.146.86.188'
TARGET_DB = 'coastshop_hour'
COUCHDB_LINK = 'http://' + TARGET_HTTP + ':5984/' + TARGET_DB
#################################################################################
# # to query the view for cities and weather
db = couchquery.Database(COUCHDB_LINK)
#################################################################################
# # calculate the percentage of positive negative netrual for each day each city
regionlist = ['coast', 'shop', 'other']
for region in regionlist:
rows = db.views.mapreduce.total_condi(key=region)
rows_other = db.views.mapreduce.total_condi(key="other")
dataframelist = []
dataframelist_other = []
#################################################################################
# # convert date to day month year hour minute second
for row, other in zip(rows, rows_other):
# row[0] = time.strptime(row[0], "%d %m %Y %H %M %S")
row[0] = datetime.datetime.strptime(row[0], "%d %m %Y %H %M %S")
row[6] = float(row[6])
row[7] = float(row[7])
row[8] = float(row[8])
row[9] = float(row[9])
other[0] = datetime.datetime.strptime(other[0],
"%d %m %Y %H %M %S")
other[6] = float(other[6])
other[7] = float(other[7])
other[8] = float(other[8])
other[9] = float(other[9])
dataframelist.append(row)
dataframelist_other.append(other)
##############################################################################
# withdraw corresponding data
df = pd.DataFrame(dataframelist,
columns=[
'date', 'positive', 'netrual', 'negative',
'tweets', 'condition', 'humidity', 'pressure',
'temperature', 'windSpeed'
])
indexeddf = df.set_index(['date'])
df_other = pd.DataFrame(dataframelist_other,
columns=[
'date', 'positive', 'netrual', 'negative',
'tweets', 'condition', 'humidity',
'pressure', 'temperature', 'windSpeed'
])
indexeddf_other = df_other.set_index(['date'])
twitterinfo = ['positive', 'netrual', 'negative', 'tweets']
weahterinfo = [
'condition', 'humidity', 'pressure', 'temperature', 'windSpeed'
]
##############################################################################
# set parameters for DBSCAN
n_neighbors = 10
n_components = 2
namelist = "hour_dimention_weahterinfo_" + region
##############################################################################
# draw cluster according to Y and compareY after mds
Y = manifold.Isomap(n_neighbors, n_components).fit_transform(
indexeddf[weahterinfo].values)
compareY = manifold.Isomap(n_neighbors, n_components).fit_transform(
indexeddf[twitterinfo].values)
drawwDBSCAN(Y, compareY, namelist)
# if tc["branch_id"]["branch_id"] not in branch_map:
# print pk, 'is not in branch map.', tc["branch_id"]["branch_id"]
# return None
def pushterm(s):
sys.stdout.write(s)
sys.stdout.flush()
if __name__ == "__main__":
if sys.argv[-1].startswith("http"):
uri = sys.argv[-1]
else:
uri = 'http://localhost:5984/tcm'
print "Database is "+uri
db = couchquery.Database(uri)
db.sync_design_doc('tcm', tcm.design_doc)
check = db.views.tcm.casesByLitmus(descending=True, limit=1)
if len(check) is not 0:
print "Latest litmus test imported was "+str(check.keys()[0])
startkey = check.keys()[0]
else:
print "Litmus pk query returned "+str(len(check))
for key in range(startkey, endkey):
if key not in skip:
result = None
while result is None:
try:
result = get_testcase(key, db)
def getTinderboxIDfromDB():
#TODO: make the db a global variable or parameter to function
# db = couchquery.Database("http://pythonesque.org:5984/fennec_test")
# db = couchquery.Database("http://*****:*****@happyhans.couch.io/logcompare")
db = couchquery.Database("http://10.2.76.100:5984/fennec_alpha")
return db.views.fennecResults.byTinderboxID()
def writeTempTweetcsv():
TARGET_HTTP = '115.146.86.188'
TARGET_DB = 'citysummary'
COUCHDB_LINK = 'http://'+TARGET_HTTP+':5984/'+TARGET_DB
#################################################################################
# # to query the view for cities and weather
db = couchquery.Database(COUCHDB_LINK)
#################################################################################
# # calculate the percentage of positive negative netrual for each day each city
rows = db.views.mapreduce.emotionTemp(key="Melbourne")
rows_other = db.views.mapreduce.emotionTemp(key="Sydney")
dataframelist = []
dataframelist_other = []
#################################################################################
# # convert date to day month year
for row,other in zip(rows,rows_other):
# row[0] = time.strptime(row[0], "%d %m %Y %H %M %S")
row[0] = datetime.datetime.strptime(row[0], "%d %b %Y")
row[4] =float(row[4])
row[5] =float(row[5])
other[0] = datetime.datetime.strptime(other[0], "%d %b %Y")
other[4] =float(other[4])
other[5] =float(other[5])
dataframelist.append(row)
dataframelist_other.append(other)
df = pd.DataFrame(dataframelist,columns=['date','positive', 'netrual', 'negative','high','low','diff','tweets','condition'])
indexeddf = df.set_index(['date'])
df_other = pd.DataFrame(dataframelist_other,columns=['date','positive', 'netrual', 'negative','high','low','diff','tweets','condition'])
indexeddf_other = df_other.set_index(['date'])
matplotlib.style.use('ggplot')
# dflookup = pd.DataFrame(indexeddf, columns = ['positive', 'netrual', 'negative','tweets','humidity','pressure','temperature','windSpeed'])
dflookup_coast = pd.DataFrame(indexeddf, columns = ['positive', 'netrual', 'negative','high','low','diff','tweets'])
dflookup_other = pd.DataFrame(indexeddf_other, columns = ['positive', 'netrual', 'negative','high','low','diff','tweets'])
# print dflookup_other
# coast_other = dflookup_coast[['positive', 'netrual', 'negative','tweets']].subtract(dflookup_other[['positive', 'netrual', 'negative','tweets']], axis=0)
# coast_other[['high','low','diff']] = dflookup_other[['high','low','diff']]
############################################################
# # correlation and convariance matrix
dflookup_coast.cov().to_csv("Sydney_day_cov.csv")
dflookup_coast.corr().to_csv("Sydney_day_corr.csv")
################################################
# # withdraw corresponding columns
dflookup_coast_tweet = pd.DataFrame(indexeddf_other, columns = ['positive', 'netrual', 'negative','tweets'])
dflookup_coast_temp = pd.DataFrame(indexeddf_other, columns = ['positive', 'netrual', 'negative','high','low','diff'])
dflookup_coast_tweet.plot(secondary_y=['tweets'])
dflookup_coast_temp.plot(secondary_y=['high','low','diff'])
###########################################################################
# # scatter matrix
scatter_matrix(dflookup_coast, alpha=0.2, figsize=(6, 6), diagonal='kde')
plt.show()
def writeTempTweetcsv():
TARGET_HTTP = '115.146.86.188'
TARGET_DB = 'coastshop_hour'
COUCHDB_LINK = 'http://' + TARGET_HTTP + ':5984/' + TARGET_DB
#################################################################################
# # to query the view for cities and weather
db = couchquery.Database(COUCHDB_LINK)
#################################################################################
# # calculate the percentage of positive negative netrual for each day each city
rows = db.views.mapreduce.totalsum(key="coast")
rows_other = db.views.mapreduce.totalsum(key="other")
dataframelist = []
dataframelist_other = []
#################################################################################
# # convert date to day month year hour minute second
for row, other in zip(rows, rows_other):
# row[0] = time.strptime(row[0], "%d %m %Y %H %M %S")
row[0] = datetime.datetime.strptime(row[0], "%d %m %Y %H %M %S")
row[6] = float(row[6])
row[7] = float(row[7])
row[8] = float(row[8])
row[9] = float(row[9])
other[0] = datetime.datetime.strptime(other[0], "%d %m %Y %H %M %S")
other[6] = float(other[6])
other[7] = float(other[7])
other[8] = float(other[8])
other[9] = float(other[9])
dataframelist.append(row)
dataframelist_other.append(other)
df = pd.DataFrame(dataframelist,
columns=[
'date', 'positive', 'netrual', 'negative', 'tweets',
'condition', 'humidity', 'pressure', 'temperature',
'windSpeed'
])
indexeddf = df.set_index(['date'])
df_other = pd.DataFrame(dataframelist_other,
columns=[
'date', 'positive', 'netrual', 'negative',
'tweets', 'condition', 'humidity', 'pressure',
'temperature', 'windSpeed'
])
indexeddf_other = df_other.set_index(['date'])
matplotlib.style.use('ggplot')
#############################################################
# # withdraw corresponding columns
dflookup_coast = pd.DataFrame(indexeddf,
columns=[
'positive', 'netrual', 'negative',
'tweets', 'humidity', 'pressure',
'temperature', 'windSpeed'
])
dflookup_other = pd.DataFrame(indexeddf_other,
columns=[
'positive', 'netrual', 'negative',
'tweets', 'humidity', 'pressure',
'temperature', 'windSpeed'
])
# print dflookup_other
coast_other = dflookup_coast[[
'positive', 'netrual', 'negative', 'tweets'
]].subtract(dflookup_other[['positive', 'netrual', 'negative', 'tweets']],
axis=0)
coast_other[['humidity', 'pressure', 'temperature',
'windSpeed']] = dflookup_other[[
'humidity', 'pressure', 'temperature', 'windSpeed'
]]
############################################################
# # correlation and convariance matrix
coast_other.cov().to_csv("coast_other_cov.csv")
coast_other.corr().to_csv("coast_other_corr.csv")
coast_other.plot()
###########################################################################
# # scatter matrix
scatter_matrix(coast_other, alpha=0.2, figsize=(6, 6), diagonal='kde')
plt.show()
import sys
import couchquery
import re
rePrivateNetworks = re.compile(
r'https?://(localhost|127\.0\.0\.1|192\.168\.[0-9]+\.[0-9]+|172\.16\.[0-9]+\.[0-9]+|10\.[0-9]+\.[0-9]+\.[0-9]+)'
)
db = couchquery.Database('http://localhost:5984/crashtest')
bad_lines = []
bad_results = []
def parse_line(line):
try:
# split the line into 16 variables. This will allow comments with
# embedded tabs to be properly parsed.
(
signature,
url,
uuid_url,
client_crash_date,
date_processed,
last_crash,
product,
version,
build,
branch,
os_name,
os_version,
