def send_post(path, args, log):
"""
Send a given body to each agent at the given path.
Encodes each body with the agent's corresponding secret
key before sending
@type path: string
@param path: The path to send the body to
@type args: JSON parsable object (dict, list, string, etc)
@param args: Body to be encoded and sent
@type log: logger
@param log: Logger object for logging
"""
global agents
lock.acquire()
try:
for x in range(0, len(agents)):
log.debug("Agent -- " + agents[x].__str__())
if agents[x]:
try:
body = util.encode(args,
agents[x].getKey()) # Encode the body
util.send_post(agents[x].getAddress() + path, body,
log) # Send the body
except:
log.warning("Agent at " + agents[x].getName() +
" is no longer alive.")
agents[x].setStatus(False)
finally:
lock.release()
def send_filters_one(agent, log):
"""
Send filters to each agent that needs it.
@type agent: Agent
@param agent: Agent to send filters to
@type log: logger
@param log: Logger object for logging
"""
global agents, filters
for filt in filters:
log.debug("Agent -- " + agent.__str__())
if agent and filters[filt].isUsed(agent.getName()):
try:
# Encode the body
body = util.encode(
json.dumps({
'code': filt,
'filter': filters[filt].getInfo()
}), agent.getKey())
util.send_post(agent.getAddress() + '/filters', body,
log) # Send the body
except:
log.warning("Agent at " + agent.getName() +
" is no longer alive.")
agent.setStatus(False)
def send_filters(log, tag=None):
"""
Send filters to each agent that needs it.
@type log: logger
@param log: Logger object for logging
@type tag: string
@param tag: None if send all filters, otherwise the tag of the specific filter to send
"""
global agents, filters
lock.acquire()
try:
for filt in filters:
if tag:
filt = tag
for x in range(0, len(agents)):
log.debug("Agent -- " + agents[x].__str__())
if agents[x] and filters[filt].isUsed(agents[x].getName()):
# try:
dump = {'code': filt, 'filter': filters[filt].getInfo()}
body = util.encode(json.dumps(dump),
agents[x].getKey()) # Encode the body
util.send_post(agents[x].getAddress() + '/filters', body,
log) # Send the body
# except:
# log.warning("Agent at " + agents[x].getAddress() + " is no longer alive.")
# agents[x].setStatus(False)
if tag:
break
finally:
lock.release()
def addSpice(mid, metadata):
try:
log('[spicemanager] in func:addSpice')
gituser = metadata['gituser']
token = metadata['token']
repo = metadata['repo']
spicename = metadata['spicename']
author = metadata['author']
tagline = metadata['tagline']
description = metadata['description']
#hotkey = metadata['hotkey']
# check to make sure that the user owns spice
spicemeta = get('metadata:%s' % mid)
if spicemeta and spicemeta['gituser'] != gituser:
raise Exception('User %s is not authorized to change this spice.' % gituser)
# clone their repo into /home/ec2-user/temp/cloned
raise Exception('Hmmmmm., how to do this....')
# Re-arrange their files
cmd('mkdir /home/ec2-user/temp/cloned/%s' % spicename)
cmd('mv /home/ec2-user/temp/cloned/* /home/ec2-user/temp/cloned/%s' % spicename)
# tar files from it to spices/
iconpath = '/home/ec2-user/temp/cloned/%s/icon.png' % spicename
if not os.path.exists(iconpath):
cmd('tar -cvf /home/ec2-user/spices/%s.tar.gz /home/ec2-user/spicerackdev/%s' % (mid, spicename))
else:
cmd('cp %s /home/ec2-user/icons/%s.png' % (iconpath, mid))
iconpath = iconpath.strip('icon.png')
cmd('tar -cvf /home/ec2-user/spices/%s.tar.gz /home/ec2-user/spicerackdev/%s --directory=%s icon.png' % (mid, spicename, iconpath))
# delete the cloned repo
cmd('rm -f -r /home/ec2-user/temp/cloned/%s') % spicename
# Send a POST to the heroku server
payload = {}
payload[mid] = {
'spicename' : spicename,
'gituser' : gituser,
'author' : author,
'tagline' : tagline,
'description' : description,
'version' : get('version:%s' % mid),
'hotkey' : hotkey,
}
put('metadata:%s' % mid, payload[mid])
send_post(payload, '/spices/refresh')
except Exception as e:
payload = {'error' : 'Error in adding Spice: %s' % str(e)}
send_post(payload, '/spices/refresh')
def del_filters(agent, filt, log):
"""
Delete the filter at the given agent.
@type agent: Agent
@param agent: Agent to contact
@type filt: string
@param filt: Name of the filter to be deleted
@type log: logger
@param log: Logger object for logging
"""
try:
body = util.encode(json.dumps({'name': filt}),
agent.getKey()) # Encode the body
util.send_post(agent.getAddress() + '/delfilter', body,
log) # Send the body
except:
log.warning("Agent at " + agent.getName() + " is no longer alive.")
agent.setStatus(False)
def init_one(agent, log):
"""
Initialize one newly recognized agent.
Runs on a thread and sends all records.
@type agent: Agent
@param agent: New agent to initialize
@type log: logger
@param log: Logger object for logging
"""
# Attempt to connect three times before giving up on the agent.
tries = 3
while tries > 0:
try:
util.send_post(agent.getAddress() + '/init',
util.encode('[]', agent.getKey()), log)
break
except:
# If could not connect, decrement remaining tries and sleep 2 seconds
tries -= 1
sleep(2)
continue
# Send all the records to the new agent and then reload its DNSMasq
log.info("Filters with " + agent.getAddress())
send_filters_one(agent, log)
log.info("NS with " + agent.getAddress())
send_records_one(agent, 'ns', ns_file, log)
log.info("A with " + agent.getAddress())
send_records_one(agent, 'a', a_file, log)
log.info("PTR with " + agent.getAddress())
send_records_one(agent, 'ptr', ptr_file, log)
log.info("Reloading " + agent.getAddress())
util.send_post(agent.getAddress() + '/reload',
util.encode('[]', agent.getKey()), log)
log.info("Done with " + agent.getAddress())
def send_records_one(agent, rtype, fl, log):
"""
Send all of the records of one type from the local file to the given agent.
@type agent: Agent
@param agent: New agent to initialize
@type rtype: string
@param rtype: Defines the records type ('ns', 'a', or 'ptr')
@type fl: string
@param fl: Address of the file
@type log: logger
@param log: Logger object for logging
"""
ret = []
for r in communicator.sendFromFile(fl):
ret.append(r)
if len(ret) > 1000: # Send 1000 records at a time
body = util.encode(json.dumps(ret), agent.getKey())
util.send_post(agent.getAddress() + '/init/' + rtype, body, log)
ret = []
# Send remaining records
body = util.encode(json.dumps(ret), agent.getKey())
util.send_post(agent.getAddress() + '/init/' + rtype, body, log)
def shakeHands(agent):
"""
Run handshake.
@type agent: Agent
@param agent: Agent to shake hands with
"""
try:
agent.generateNewKey()
agent.generateKey(
util.send_post(agent.getAddress() + '/handshake',
agent.getPublicKey(), log))
log.debug("Handshake complete. New secret key for " + agent.getName())
except:
log.warning("Agent at " + agent.getName() + " is no longer alive.")
agent.setStatus(False)
errHandler.setLevel(logging.ERROR)
# Create stream handler for logging to the console
if logToConsole:
stdHandler = logging.StreamHandler()
stdHandler.setFormatter(logging.Formatter(logFormat))
stdHandler.setLevel(logLevel)
log.addHandler(stdHandler)
# Add handlers to log instance
log.addHandler(mainHandler)
log.addHandler(errHandler)
try:
# try:
key.generateKey(util.send_post(home + '/handshake',{'name': name, 'port': port, 'key': key.publicKey}, log))
log.debug(key.getKey())
log.info("Handshake complete. Secret key generated.")
util.send_get(home + '/init', {'name': name, 'port': port}, log)
# except URLError, err:
# log.warning("Could not call home: " + str(err.reason))
reactor.listenTCP(port, server.Site(HelloResource()))
log.info("Server started on port " + str(port))
reactor.run()
except:
error = 'Failure: ' + str(sys.exc_info()[0])
log.critical(error)
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
error = ''.join('!! ' + line for line in lines)
def spiceMerge(spices, username):
"""
Manages the merger of a spice into the development repository.
@param spices: list of (spicename, dirname)
"""
log('[spicemanager] In func:spiceMerge')
goto_dir(makepath('spicerackdev'))
command('git', 'pull')
goto_dir(makepath(''))
spice_to_metadata = {}
for dirname in spices:
data = jsonload(makepath('spicerackdev', dirname, 'metadata.json'))
spicename = data['name']
if 'spicename' not in data:
data['spicename'] = spicename
if 'width' not in data:
data['width'] = 0
data['height'] = 0
data['dependencies'] = []
Statsd.increment('spices.spicerackdev.%s' % spicename.replace(' ', '-'))
mid = get('spice-id:%s' % spicename.replace(' ', '-'))
if mid is None:
mid = str(uuid.uuid4())
put('spice-id:%s' % spicename.replace(' ', '-'), mid)
log('[spicemanager] Processing %s %s' % (spicename, mid))
version = 1 + int(dget('version:%s' % mid, 0))
put('version:%s' % mid, version)
log('[spicemanager] Version: %s' % version)
spice_to_metadata[mid] = {
'spicename' : data['spicename'],
'gituser' : username,
'author' : data['author'],
'tagline' : data['description'],
'description' : data['description'],
'version' : version,
'hotkey' : False,
'main' : data['main'],
'width' : data['width'],
'height' : data['height'],
'dependencies' : data['dependencies']
}
put('metadata:%s' % mid, spice_to_metadata[mid])
jsondump(makepath('spicerackdev', dirname, 'metadata.json'), spice_to_metadata[mid])
# Tar the spice directory into /spices/{mid}.tar.gz
command('tar', '-cvf', makepath('spices', '%s.tar.gz' % mid), makepath('spicerackdev', dirname))
# Copy the icon file into /icons
iconpath = makepath('spicerackdev', dirname, 'icon.png')
command('cp', '-f', iconpath, makepath('icons', '%s.png' % mid))
iconpath = makepath(dirname, 'icon.png')
if not os.path.exists(iconpath):
command('tar', '-cvf', makepath('spices', '%s.tar.gz' % mid), makepath('spicerackdev', dirname))
else:
iconpath = iconpath.strip('icon.png')
command('tar',
'-cvf',
makepath('spices', '%s.tar.gz' % mid),
makepath('spicerackdev', dirname),
'--directory=%s' % iconpath,
'icon.png')
goto_dir(makepath('spicerackdev'))
command('git', 'stash')
goto_dir(makepath(''))
heroku_metadata = {
'spicename' : data['spicename'],
'gituser' : username,
'author' : data['author'],
'tagline' : data['description'],
'description' : data['description'],
'version' : version,
'hotkey' : False,
'main' : data['main']
}
log('HEROKU: %s %s', type(heroku_metadata), heroku_metadata)
for key, val in heroku_metadata.iteritems():
log('KEY %s VAL %s %s', key, type(val), val)
resp, content = send_post({str(mid) : heroku_metadata}, '/spices/refresh')
log('[spicemanager] Heroku response: %s' % str(resp))
return 'success'
def spiceMerge(spices, username):
"""
Manages the merger of a spice into the development repository.
@param spices: list of (spicename, dirname)
"""
log('[spicemanager] In func:spiceMerge')
chdir('/home/ec2-user/spicerackdev')
cmd('git pull')
chdir('/home/ec2-user')
payload = {}
for dirname in spices:
with open('/home/ec2-user/spicerackdev/%s/metadata.json' % dirname) as f:
data = json.load(f)
spicename = data['name'] if 'name' in data else data['spicename']
mid = get('spice-id:%s' % spicename.replace(' ', '-'))
log('[spicemanager] Processing %s %s' % (spicename, mid))
log('[spicemanager] Spice %s has mid %s' % (spicename.replace(' ', '-'), mid))
if mid is None:
mid = str(uuid.uuid4())
put('spice-id:%s' % spicename.replace(' ', '-'), mid)
version = 1 + int(dget('version:%s' % mid, 0))
put('version:%s' % mid, version)
log('[spicemanager] Version: %s' % version)
payload[mid] = {
'spicename' : data['name'] if 'name' in data else data['spicename'],
'gituser' : username,
'author' : data['author'],
'tagline' : data['description'],
'description' : data['description'],
'version' : version,
'hotkey' : False,
'main' : data['main']
}
if 'height' in data:
payload[mid]['height'] = data['height']
if 'width' in data:
payload[mid]['width'] = data['width']
put('metadata:%s' % mid, payload[mid])
with open('/home/ec2-user/spicerackdev/%s/metadata.json' % dirname, 'w') as f:
json.dump(payload[mid], f)
if 'height' in payload[mid]:
del payload[mid]['height']
if 'width' in payload[mid]:
del payload[mid]['width']
# Tar the spice directory into /spices/{mid}.tar.gz
cmd('tar -cvf /home/ec2-user/spices/%s.tar.gz /home/ec2-user/spicerackdev/%s' % (mid, dirname))
# Copy the icon file into /icons
iconpath = '/home/ec2-user/spicerackdev/%s/icon.png' % dirname
cmd('cp -f %s /home/ec2-user/icons/%s.png' % (iconpath, mid))
iconpath = '/home/ec2-user/%s/icon.png' % dirname
if not os.path.exists(iconpath):
cmd('tar -cvf /home/ec2-user/spices/%s.tar.gz /home/ec2-user/spicerackdev/%s' % (mid, dirname))
else:
iconpath = iconpath.strip('icon.png')
cmd('tar -cvf /home/ec2-user/spices/%s.tar.gz /home/ec2-user/spicerackdev/%s --directory=%s icon.png' % (mid, dirname, iconpath))
chdir('/home/ec2-user/spicerackdev')
cmd('git stash')
chdir('/home/ec2-user')
resp, content = send_post(payload, 'spices/refresh')
log('[spicemanager] Heroku response: %s' % str(resp))
return 'success'