def commit_set_company (commit):
co = None
author = commit['author'].lower()
author_date = int(commit['author_date'])
email = commit['author_email'].lower()
# Check email's domain
for company in DOMAIN_MATCH:
matched = DOMAIN_MATCH[company].findall (email)
if matched:
commit['company'] = company
return commit
# Check author's email addresses
if (('vishvananda' in email) or
('anthony young' in author) or
('jason koelker' in author or 'jason köelker' in author)):
co = 'Rackspace'
elif 'joshua mckenty' in author:
co = 'NASA'
elif 'jesse andrews' in author:
if author_date < utils.date_to_unix(2011,02):
co = 'NASA'
elif author_date < utils.date_to_unix(2011,07):
co = 'Rackspace'
elif author_date < utils.date_to_unix(2012,07):
co = 'Nebula'
elif 'gabriel hurley' in author:
if author_date < utils.date_to_unix(2011,07):
co = 'NASA'
else:
co = 'Nebula'
elif 'devin carlen' in author:
if author_date < utils.date_to_unix(2011,03):
co = 'NASA'
else:
co = 'Nebula'
elif 'jay pipes' in author:
if author_date < utils.date_to_unix(2011,12):
co = 'Rackspace'
elif author_date < utils.date_to_unix(2012,06):
co = 'HP'
else:
co = 'AT&T'
elif 'yun mao' in author:
co = 'AT&T'
elif 'rick harris' in author:
co = 'VMWare'
elif 'alessandro pilotti' in author:
co = 'CloudBase'
elif 'william wolf' in author:
co = 'Crowd Tilt'
# A few special cases
commit['company'] = co
return commit
def publish_data( keyvals, msg_dbg=''):
if keyvals != None and 'driver' in keyvals and 'device_id' in keyvals:
driver = keyvals['driver']
device_id = keyvals['device_id']
devdef = utils.read_device(driver)
if devdef == None:
print "Device definition for %s not found"%driver
return
## Custom device drivers here
if driver == 'custom!@#!@$!$#':
pass
else: # generic driver: try to match keys to feeds
if 'timestamp' in keyvals:
dev = publisher.find_device( device_id, create_new=True, device_type=driver, devdef=devdef )
ts = utils.date_to_unix(utils.str_to_date(keyvals['timestamp']))
datapoints = []
feednums = []
for key,val in keyvals.iteritems():
if key in ['driver','device_id','timestamp']:
continue
try:
f = float(val);
except:
print "Skipping Key-Value pair",key,"/",val,"as it is non-numeric"
continue;
if key in dev.feed_names:
idx = dev.feed_names.index(key)
feednums.append(idx)
datapoints.append(f)
else:
feednums.append(len(dev.feed_names))
datapoints.append(f)
dev.feed_names.append(key)
try:
if (len(datapoints) > 0):
publisher.publish_data( device_id, ts, datapoints, feednum=feednums, devdef=devdef, device_type=driver, dev=dev)
except:
import traceback
traceback.print_exc();
else:
print "Data Line '%s' did not have a timestamp field (for generic driver)"%msg_dbg
else:
print "Data Line '%s' did not contain a key for driver and/or device_id"%msg_dbg
tto = DT.datetime.now()
f = tempfile.NamedTemporaryFile(delete = False)
fname = f.name
f.close()
fcsv = tempfile.NamedTemporaryFile(delete = False)
fcsvname = fcsv.name
fcsv.close()
if 'title' in d:
titlearg = "--title \""+d['title'][0]+"\""
else:
titlearg = ""
cmdline = str(config.map['web']['fetchcmd'] + " fetch --plot %s %s --from %d --to %d %s %s %s"%(fname,titlearg,utils.date_to_unix(tfrom),utils.date_to_unix(tto),source,sourceid,fcsvname))
print cmdline
cmds = shlex.split(cmdline)
subprocess.call(cmds)
if os.path.exists(fcsvname):
os.unlink(fcsvname);
if os.path.getsize(fname) == 0:
raise Exception("Plot file not created. Error plotting?")
start_response('200 OK',[('Content-Type','image/png')])
return TempFileWrapper(fname);
except Exception,exp:
import traceback
def run(self, time_from, time_to, source_name=None, source_id=None, pretend=False, use_cache=True, local=False, params=[]):
if source_id == None and local:
print "Error: Can only run 'local' test on one stream ( source name & id pair ) at a time"
sys.exit(1)
if source_name == None:
names = utils.list_sources();
for name in names:
ids = utils.list_ids(name)
for id in ids:
self.run(time_from, time_to, name, id, pretend, use_cache, local)
return
elif source_id == None:
ids = utils.list_ids(source_name)
for id in ids:
self.run(time_from, time_to, source_name, id, pretend, use_cache, local)
return
smap = utils.read_source(source_name)
if not local and use_cache:
# find any cached files
import filedb
filedb.connect()
for file in self.files:
stepchain = file.stepchain
cfiles = filedb.get_files(where='time_from=%s and time_to=%s and source_name=%s and source_id=%s and steps=%s',params=(time_from,time_to,source_name,source_id,stepchain))
for cfile in cfiles:
if ( cfile.status == filedb.INVALID or os.path.exists(cfile.file_name) ):
file.cached = True
file.file_id = cfile.id
file.fname = cfile.file_name
file.deptask = cfile.task_id
file.stepchain = stepchain
print "Found cached file for output of "+file.src.name
break
# prune any tasks that don't need to be run
for step in self.steps[:]:
if ( len(step.outputs) == 0):
print "Step %s will be run b/c it has no outputs"%step.name
continue
canbepruned = True
for f in step.outputs:
if not f.cached:
canbepruned = False
break
if canbepruned:
print "Pruning step %s because the cache can supply all outputs"%step.name
self.steps.remove(step)
else:
for f in step.outputs:
if f.cached:
f.cached = False
print "Cached file %s.O%d will be regenerated b/c not all outputs were cached"%(step.name,f.index)
# create all the files we'll need
if local:
dir = ''
else:
if ( self.use_tmp ):
dir = '/tmp/sensezilla/flowdata/'+self.name
else:
dir = config.map['global']['data_dir']+'/flowdata/'+self.name
if not pretend:
try:
os.makedirs(dir+'/outputs',0755)
except:pass
for file in self.files:
if not file.cached:
if local:
file.fname = dir+'testing_%s_%s.O%d'%(self.name,file.src.name,file.index)
if not pretend:
if file.directory:
try:
os.mkdir(file.fname)
except:pass
else:
fout = open(file.fname,'w')
fout.close()
else:
if 'OUTPUT' not in [v[0] for v in file.dests]:
if not pretend:
if file.directory:
file.fname = tempfile.mkdtemp(dir=dir)
else:
tfile = tempfile.NamedTemporaryFile('w', dir=dir, delete=False)
file.fname = tfile.name;
tfile.close()
else:
file.fname = os.tempnam(dir)
else:
file.fname = dir+'/outputs/%s.O%d_%s_%s_%d_to_%d'%(file.src.name,file.index,source_name,source_id.replace('/','.'),
utils.date_to_unix(time_from),utils.date_to_unix(time_to))
if file.directory:
if not pretend:
os.mkdir(file.fname)
else:
if not pretend:
fout = open(file.fname,'w')
fout.close()
if file.directory:
print "Created directory : "+file.fname
else:
print "Created file : "+file.fname
# generate dictionary of substitutions
subs = {
'TIME_FROM':int(utils.date_to_unix(time_from)),
'TIME_TO':int(utils.date_to_unix(time_to)),
'SOURCE':source_name,
'ID':source_id
};
subs.update(params)
try:
import devicedb
devicedb.connect()
plmeta,dev,pl_index = devicedb.find_plugload(source_name,source_id)
subs['PLUGLOAD'] = plmeta.value;
subs['DEVID'] = dev.ID
subs['DEVIDSTR'] = dev.IDstr
except Exception,e:
print "Cannot contact devicedb "+str(e)