def appendBhavData(base, bhavDataLoc):
#Appends new information to existing bhav-data-base#
with open(base,'r') as f:
ext = pickle.load(f)
newData = fl.readLinesAndSplit(bhavDataLoc, ',')
for line in newData:
if str(line[1])+'_'+str(line[2]) not in uniqCName:
uniqCName.append(str(line[1])+'_'+str(line[2]))
for cName in uniqCName:
if cName not in ext.keys():
ext[cName] = {'t-price': [], 't-vol': [], 't-share': []}
for line in data:
if line[0] != "DATE":
if cName.split('_')[0] == line[1] or cName.split('_')[1] == line[2]:
ext[cName]['t-price'].append((line[0], line[8]))
ext[cName]['t-vol'].append((line[0], line[11]))
ext[cName]['t-share'].append((line[0], line[12]))
return ext
def parseBhavData(bhavDataLoc):
#Identifies consolidated Bhav report (2y) and parses company-wise data. Other functions are defined in mathutils#
print "Extracting BHAV data"
data = fl.readLinesAndSplit(bhavDataLoc, ',')
TS_DATA = {}
uniqCName = []
for line in data:
if str(line[1])+'_'+str(line[2]) not in uniqCName:
uniqCName.append(str(line[1])+'_'+str(line[2]))
for cName in uniqCName:
TS_DATA[cName] = {'t-price': [], 't-vol': [], 't-share': []}
for line in data:
if line[0] != "DATE":
if cName.split('_')[0] == line[1] or cName.split('_')[1] == line[2]:
TS_DATA[cName]['t-price'].append((line[0], line[8]))
TS_DATA[cName]['t-vol'].append((line[0], line[11]))
TS_DATA[cName]['t-share'].append((line[0], line[12]))
return TS_DATA
def createSiteMap(fileLoc):
data = fl.readLinesAndSplit(fileLoc, '/')
#Creating nXm matrix
max_depth = max([len(x) for x in data])
Rel = []
for line in data:
if len(line) > 2:
for j in range(len(line)-1):
if line[j] != '' and line[j+1] != '':
x = codify(str(line[j]), '-') if len(line[j]) > 15 else line[j]
y = codify(str(line[j+1]), '-') if len(line[j+1]) > 15 else line[j+1]
Rel.append(str(x)+'--'+str(y))
with open('cluster','w') as f:
for line in Rel:
f.write(str(line)+'\n')
return Rel
def createHeatMap(fileLoc):
data = fl.readLinesAndSplit(fileLoc, ',')
data.pop(0)
types = list(set([v[0] for v in data]))
connect = {}
Rel = {}
#Calculating page view stats
pv_max = max([v[3] for v in data])
pv_min = min([v[3] for v in data])
pt_max = max([v[4] for v in data])
pt_min = min([v[4] for v in data])
for tp in types:
Rel[tp] = {}
for line in data:
if line[0] == tp:
x = codify(str(line[1]), '/') if len(line[1]) > 15 else line[1]
y = codify(str(line[2]), '/') if len(line[2]) > 15 else line[2]
w = (1+int(line[3]))*(1+int(line[4]))
if x != y:
if str(x)+'--'+str(y) in Rel[tp]:
Rel[tp][str(x)+'->'+str(y)] += w
else:
Rel[tp][str(x)+'->'+str(y)] = w
for tp in types:
r_mean = np.mean(Rel[tp].values())
r_std = np.std(Rel[tp].values())
colored = []
with open('hc_'+str(tp), 'w') as f:
for elem in Rel[tp]:
w = ((Rel[tp][elem] - r_mean)/r_std) + 0.4
print w
f.write(str(elem)+' {color:'+colorCode(w)+'}\n')
if elem.split('->')[1] not in colored:
f.write(str(elem.split('->')[1])+' {color:'+colorCode(w)+'}\n')
colored.append(elem.split('->')[1])