def test_disable_pyparsing_arity_trimming_works():
"""Tests that arity trimming has been disabled and parse actions with
the wrong number of arguments will raise TypeErrors"""
for func in [lambda a: None, lambda a, b: None, lambda a, b, c, d: None]:
element = Literal('test').setParseAction(func)
with raises(TypeError):
element.parseString('test')
def test_disable_pyparsing_arity_trimming_works():
"""Tests that arity trimming has been disabled and parse actions with
the wrong number of arguments will raise TypeErrors"""
for func in [lambda a: None, lambda a, b: None, lambda a, b, c, d: None]:
element = Literal('test').setParseAction(func)
with raises(TypeError):
element.parseString('test')
def check_unnecessary_include(self, code):
grammar = Literal('#') + Literal('include') + Literal('<') + Word(alphanums)
try:
grammar.parseString(code)
begin = code.find("<")
end = code.find(">")
included_library = code[begin + 1:end]
if included_library not in self.includes:
self.add_error(label="UNNECESSARY_INCLUDE")
except ParseException:
return
def check_local_include(self, code):
grammar = Literal('#') + Literal('include') + Literal('"') + Word(alphanums)
try:
grammar.parseString(code)
begin = code.find('"')
included_file = code[begin + 1:]
end = included_file.find('"')
included_file = included_file[:end]
if included_file not in self.includes:
self.local_includes[self.current_file].append(included_file)
except ParseException:
return
def check_local_include(self, code):
grammar = Literal('#') + Literal('include') + Literal('"') + Word(
alphanums)
try:
grammar.parseString(code)
begin = code.find('"')
included_file = code[begin + 1:]
end = included_file.find('"')
included_file = included_file[:end]
if included_file not in self.includes:
self.local_includes[self.current_file].append(included_file)
except ParseException:
return
def check_unnecessary_include(self, code):
grammar = Literal('#') + Literal('include') + Literal('<') + Word(
alphanums + '.' + '_') + Literal('>')
try:
grammar.parseString(code)
begin = code.find("<")
end = code.find(">")
included_library = code[begin + 1:end]
if included_library not in self.includes:
self.add_error(label="UNNECESSARY_INCLUDE",
data={"library": included_library})
except ParseException:
return
def readFile():
tempo = []
''' READING FILE'''
fd = open('times.pl', 'r')
buffer = fd.read().split('\n')
'''Parsing lines'''
'''Diferentes tipos de parsers'''
firstLine = Literal('timetable') + Suppress('(') + Word(alphas) + Suppress(
',') + Word(alphas) + Suppress(',')
carreiraLine = Optional(
Suppress('[')) + Word(nums + ':' + nums) + Suppress('/') + Word(
nums + ':' +
nums) + Suppress('/') + Word(alphanums) + Suppress('/') + (
Literal('alldays') ^
(Suppress('[') + OneOrMore(Word(alphas + ',') ^ Word(alphas)) +
Suppress(']'))) + (Suppress(']).') ^ Suppress(','))
'''Inicio de parser'''
i = 0
index_tempo = 0
while not (buffer == [''] or buffer == []):
aux = firstLine.parseString(buffer[0])
tempo.append(Percursos(aux[1], aux[2]))
del buffer[0]
while not buffer[0] == '':
buffer[0] = buffer[0].replace(' ', '')
aux = carreiraLine.parseString(buffer[0])
tempo[index_tempo].viagens.append(Carreiras(*aux))
del buffer[0]
del buffer[0]
index_tempo += 1
return tempo
def parse_common_block(s: str) -> List[str]:
"""Parse a common block."""
myword = Word(alphanums + "_")
inside = OneOrMore(myword + ZeroOrMore("*") + ZeroOrMore(","))
parenthesis = ZeroOrMore(Char("(") + inside + Char(")"))
parser = Literal("common") + Char('/') + myword + Char('/') + \
OneOrMore(Group(myword + parenthesis + ZeroOrMore(Char(","))))
return parser.parseString(s).asList()
class Bionames (Operator):
def __init__(self):
""" Initialize the operator. """
super(Bionames, self).__init__("bionames")
self.url = "https://bionames.renci.org/lookup/{input}/{type}/"
self.select_grammar = \
Literal("select") + White() + ( QuotedString('"') | Word( alphas+"_"+"$", alphanums+"_" ) ) + \
White() + Literal("from") + White() + Word(alphas) + White() + Literal("as") + White() + Word(alphas+"_")
self.select_grammar = Literal("select") + White() + (QuotedString('"')|Word(alphas+"$"+"_")) + White() + Literal("from") + White() + Word(alphas+"_") + White() + Literal("as") + White() + Word(alphas+"_")
def execute_query (self, query, context):
#Literal("select") + White() + ( QuotedString('"') | Word( alphas+"_"+"$", alphanums+"_" ) ) + White() + Literal("from") + White() + Word(alphas)
print (f"parsing query {query}")
vals = self.select_grammar.parseString (query)
print (vals)
select_key, w0, input_string, w1, from_key, w2, category, w3, as_key, w4, name = self.select_grammar.parseString (query) #'select "this is a string" from bob')
# logger.debug (f"
context.set_result (name,
self.get_ids (
name = context.resolve_arg(input_string),
type_name = category))
def get_ids (self, name, type_name):
url = self.url.format (**{
"input" : name,
"type" : type_name
})
logger.debug (f"url: {url}")
result = None
response = requests.get(
url = url,
headers = {
'accept': 'application/json'
})
if response.status_code == 200 or response.status_code == 202:
result = response.json ()
else:
raise ValueError (response.text)
logger.debug (f"bionames result {result}")
return result
def invoke (self, event):
result = {}
if event.query:
if isinstance(event.query, str):
self.execute_query (q, event.context)
elif isinstance(event.query, list):
for q in event.query:
self.execute_query (q, event.context)
else:
result = self.get_ids (
name=event.input,
type_name=event.type)
return result
def typeSwitch(line):
global typedversion
typeflag = Literal("#") + "option" + Literal("=") + oneOf("untyped", "typed")
res = typeflag.parseString(line)
if res[3] == "untyped":
typedversion = False
elif res[3] == "typed":
typeversion = True
else:
print "Cannot determine whether typed or untyped."
raise ParseException
str = "Detected "
if not typedversion:
str += "un"
str += "typed version."
print str
def typeSwitch(line):
global typedversion
typeflag = Literal("#") + "option" + Literal("=") + oneOf(
"untyped", "typed")
res = typeflag.parseString(line)
if res[3] == "untyped":
typedversion = False
elif res[3] == "typed":
typeversion = True
else:
print "Cannot determine whether typed or untyped."
raise ParseException
str = "Detected "
if not typedversion:
str += "un"
str += "typed version."
print str
def FromString(cls, desc):
"""Parse this stop condition from a string representation.
The string needs to match:
run_time number [seconds|minutes|hours|days|months|years]
Args:
desc (str): The description
Returns:
TimeBasedStopCondition
"""
parse_exp = Literal(u'run_time').suppress() + time_interval(u'interval')
try:
data = parse_exp.parseString(desc)
return TimeBasedStopCondition(data[u'interval'][0])
except ParseException:
raise ArgumentError(u"Could not parse time based stop condition")
# Copyright L.P.Klyne 2013
# Licenced under 3 clause BSD licence
from pyparsing import Literal, Empty, replaceWith
def insertResult(v):
"""
Parser helper function that simply inserts a result in
the list of values returned.
"""
return Empty().setParseAction( replaceWith(v) )
p1 = Literal("1")
p2 = Literal("2")+insertResult("B") # 'AttributeError: 'NoneType' object has no attribute 'streamline''
p3 = insertResult("B")+Literal("3") # Blows python stack
r1 = p1.parseString("1")
r2 = p2.parseString("2")
r3 = p3.parseString("3")
print r2
print r3
date=datetime.datetime.strptime(site_contents[begin_date:end_date].strip().replace(',',''), '%b %d %Y')
except ValueError:
date=datetime.datetime.strptime(site_contents[begin_date:end_date].strip().replace(',',''), '%B %d %Y')
date=date-datetime.timedelta(hours=24)
start_index = site_contents.find('Sows Purchased (Live and Carcass Basis)')
labels = [ '300-399', '400-449', '450-499', '500-549', '550/up' ]
x = 0
parsed = []
# Loops through each label in labels and parses its line of data on the website.
# Then it creates a table with the parsed data elements and moves to the next label.
while x < len(labels):
label_index = site_contents.find(labels[x], start_index) # index of labels[x] on the website
#grammar for each line of data
line_grammar = Literal(labels[x]) + Word(nums+',') + Word(nums) + Word(nums+'.'+'-') + Word(nums+'.')
line_end = site_contents.find('\r\n', label_index) # index of the end of the line to be parsed
parsed = line_grammar.parseString(site_contents[label_index:line_end]).asList() # parses line and converts to list
parsed.append(parsed[4]) # add the weighted average to end of the list because split on next line will overwrite parsed[4]
[ parsed[3], parsed[4] ] = parsed[3].split('-') # split the price range into low price and high price
headings = [ 'Date', 'Head Count', 'Avg Wgt', 'Low Price', 'High Price', 'Wtd Avg Price' ]
data = { 'Date': [date.strftime('%Y-%m-%d')], 'Head Count': [parsed[1]], 'Avg Wgt': [parsed[2]], 'Low Price': [parsed[3]], \
'High Price': [parsed[4]], 'Wtd Avg Price': [parsed[5]] }
data_df = pd.DataFrame(data, columns = headings)
data_df.index = data_df['Date']
data_df = data_df.drop('Date', 1)
quandl_code = 'USDA_LM_HG230_' + parsed[0].replace('-', '_').replace('/', '_') + '\r'# build unique quandl code
reference_text = ' Historical figures from USDA can be verified using the LMR datamart located ' \
'\n at http://mpr.datamart.ams.usda.gov.\n'
print 'code: ' + quandl_code + '\n'
print 'name: National Daily Sows Purchased- ' + parsed[0] + ' pounds\n'
print 'description: National daily direct sow and boar report. This dataset contains '\
' head count, average weight, price range, and weighted average for sows in the weight range ' + parsed[0] +\
class Manager(object):
'''
A manager to orchestrate the creation and
deletion of container clusters
'''
def __init__(self, logger):
self.salt_client = salt.client.LocalClient()
self.etcd = Etcd(logger)
self.logger = logger
# Parse out the username and formation name
# from the ETCD directory string
self.formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
def fqdn_to_shortname(self, fqdn):
if '.' in fqdn:
return fqdn.split('.')[0]
else:
return fqdn
def check_salt_key_used(self, hostname):
self.logger.info(
"Checking if the key for {host} is already used".format(
host=hostname))
s = subprocess.Popen('salt-key', shell=True, stdout=PIPE)
salt_list = s.communicate()[0]
if hostname in salt_list:
return True
else:
return False
def check_port_used(self, host, port):
self.logger.info(
"Checking if {port} on {host} is open with salt-client".format(
host=host, port=port))
results = self.salt_client.cmd(
host,
'cmd.run',
['netstat -an | grep %s | grep tcp | grep -i listen' % port],
expr_form='list')
self.logger.debug("Salt return: {lsof}".format(lsof=results[host]))
if results[host] is not '':
return True
else:
return False
# TODO
def check_for_existing_formation(self, formation_name):
# If the user passed in an existing formation name lets append to it
pass
def get_docker_cluster(self):
# Return a list of docker hosts
cluster = self.etcd.get_key('docker_cluster')
if cluster is not None:
return cluster.split(',')
else:
return None
def get_load_balancer_cluster(self):
# Return a list of nginx hosts
cluster = self.etcd.get_key('nginx_cluster')
if cluster is not None:
return cluster.split(',')
else:
return None
def order_cluster_by_load(self, cluster_list):
# Sample salt output
# {'dlceph01.drwg.local': '0.27 0.16 0.15 1/1200 26234'}
# define grammar
point = Literal('.')
number = Word(nums)
floatnumber = Combine(number + point + number)
float_list = OneOrMore(floatnumber)
results = self.salt_client.cmd(','.join(cluster_list),
'cmd.run', ['cat /proc/loadavg'],
expr_form='list')
load_list = []
self.logger.debug("Salt load return: {load}".format(load=results))
for host in results:
host_load = results[host]
match = float_list.parseString(host_load)
if match:
one_min = match[0]
five_min = match[1]
fifteen_min = match[2]
self.logger.debug(
"Adding Load({host}, {one_min}, {five_min}, {fifteen_min}".
format(host=host,
one_min=one_min,
five_min=five_min,
fifteen_min=fifteen_min))
load_list.append(Load(host, one_min, five_min, fifteen_min))
else:
self.logger.error("Could not parse host load output")
# Sort the list by fifteen min load
load_list = sorted(load_list, key=lambda x: x.fifteen_min_load)
for load in load_list:
self.logger.debug("Sorted load list: " + str(load))
return load_list
# Retun a list of formations the user owns
def list_formations(self, username):
formation_list = []
formations = self.etcd.list_directory('formations/' + username)
for formation in formations:
parse_results = self.formation_parser.parseString(formation)
if parse_results:
formation_name = parse_results['formation_name']
formation_list.append(formation_name)
else:
self.logger.error("Could not parse the ETCD string")
self.logger.info('Formation list {formations} for user {user}'.format(
formations=formation_list, user=username))
return formation_list
# Load the formation and return a Formation object
def load_formation_from_etcd(self, username, formation_name):
f = Formation(username, formation_name)
app_list = json.loads(
json.loads(
self.etcd.get_key(
'/formations/{username}/{formation_name}'.format(
username=username, formation_name=formation_name))))
for app in app_list:
# If our host doesn't support swapping we're going to get some garbage
# message in here
if "WARNING" in app['container_id']:
app['container_id'] = app['container_id'].replace("WARNING: Your "\
"kernel does not support memory swap capabilities. Limitation discarded.\n","")
#Message changed in docker 0.8.0
app['container_id'] = app['container_id'].replace("WARNING: WARNING:"\
"Your kernel does not support swap limit capabilities. Limitation "\
"discarded.\n","")
app['container_id'].strip('\n')
# Set volumes if needed
volumes = None
if app['volumes']:
self.logger.info("Setting volumes to: " +
''.join(app['volumes']))
volumes = app['volumes']
f.add_app(app['container_id'], app['hostname'], app['cpu_shares'],
app['ram'], app['port_list'], app['ssh_port'], 22,
app['host_server'], volumes)
# Return fully parsed and populated formation object
return f
def save_formation_to_etcd(self, formation):
name = formation.name
username = formation.username
self.etcd.set_key(
'formations/{username}/{formation_name}'.format(
username=username, formation_name=name), formation)
# TODO write code to add new apps to load balancer
def add_app_to_nginx(self, app):
pass
# TODO write code to add new apps to the load balancer
def add_app_to_apache(self, app):
pass
def start_application(self, app):
# Run a salt cmd to startup the formation
docker_command = "docker run -c={cpu_shares} -d -i -t -h=\"{hostname}\" -m={ram}m "\
"--name={hostname} {port_list} {volume_list} {image} /sbin/my_init -- bash"
self.logger.info("Port list %s" % app.port_list)
port_list = ' '.join(map(lambda x: '-p ' + x, app.port_list))
# Only create this list if needed
volume_list = ''
if app.volume_list:
volume_list = ' '.join(map(lambda x: '-v ' + x, app.volume_list))
d = docker_command.format(cpu_shares=app.cpu_shares,
hostname=app.hostname,
ram=app.ram,
image=app.docker_image,
port_list=port_list,
volume_list=volume_list)
self.logger.info(
"Starting up docker container on {host_server} with cmd: {docker_cmd}"
.format(host_server=app.host_server, docker_cmd=d))
salt_process = self.salt_client.cmd(app.host_server,
'cmd.run', [d],
expr_form='list')
container_id = salt_process[app.host_server]
if container_id:
if "WARNING" in container_id:
container_id = container_id.replace("WARNING: "\
"Your kernel does not support swap limit capabilities. Limitation "\
"discarded.\n","")
container_id.strip("\n")
#Docker only uses the first 12 chars to identify a container
app.change_container_id(container_id[0:12])
def bootstrap_application(self, app):
# Log into the host with paramiko and run the salt bootstrap script
host_server = self.fqdn_to_shortname(app.host_server)
self.logger.info(
"Bootstrapping {hostname} on server: {host_server} port: {port}".
format(hostname=app.hostname,
host_server=host_server,
port=app.ssh_port))
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=host_server,
port=app.ssh_port,
username='******',
password='******')
transport = paramiko.Transport((host_server, app.ssh_port))
transport.connect(username='******', password='******')
sftp = paramiko.SFTPClient.from_transport(transport)
sftp.put('bootstrap.sh', '/root/bootstrap.sh')
sftp.put('start.sh', '/root/start.sh')
ssh.exec_command("chmod +x /root/bootstrap.sh")
ssh.exec_command("chmod +x /root/start.sh")
stdin, stdout, stderr = ssh.exec_command("bash /root/start.sh")
self.logger.debug(''.join(stdout.readlines()))
ssh.close()
except SSHException:
self.logger.error(
"Failed to log into server. Shutting it down and cleaning up the mess."
)
self.delete_container(app.host_server, app.container_id)
# Stops and deletes a container
def delete_container(self, host_server, container_id):
results = self.salt_client.cmd(
host_server,
'cmd.run',
['docker stop {container_id}'.format(container_id=container_id)],
expr_form='list')
self.logger.debug(
"Salt return: {stop_cmd}".format(stop_cmd=results[host_server]))
results = self.salt_client.cmd(
host_server,
'cmd.run',
['docker rm {container_id}'.format(container_id=container_id)],
expr_form='list')
self.logger.debug(
"Salt return: {rm_cmd}".format(rm_cmd=results[host_server]))
# Stops and deletes a formation. Use with caution
def delete_formation(self, user, formation_name):
formation_list = self.list_formations(user)
if formation_name in formation_list:
pass
else:
self.logger.error("Formation name not found!")
def list_containers(self, user, formation_name):
pass
def create_containers(self,
user,
number,
formation_name,
cpu_shares,
ram,
port_list,
hostname_scheme,
volume_list,
docker_image,
force_host_server=None):
f = Formation(user, formation_name)
# Convert ram to bytes from MB
ram = ram * 1024 * 1024
# Get the cluster machines on each creation
cluster_list = self.get_docker_cluster()
circular_cluster_list = CircularList(
self.order_cluster_by_load(cluster_list))
# Loop for the requested amount of containers to be created
for i in range(1, number + 1):
# [{"host_port":ssh_host_port, "container_port":ssh_container_port}]
ssh_host_port = 9022 + i
ssh_container_port = 22
host_server = circular_cluster_list[i].hostname
hostname = '{hostname}{number}'.format(hostname=hostname_scheme,
number=str(i).zfill(3))
# First check if we can add this host to salt. If not exit with -1
if self.check_salt_key_used(hostname):
self.logger.error(
'Salt key is already taken for {hostname}'.format(
hostname=hostname))
sys.exit(-1)
# We are being asked to overwrite this
if force_host_server:
host_server = force_host_server
validated_ports = []
while self.check_port_used(host_server, ssh_host_port):
ssh_host_port = ssh_host_port + 1
for port in port_list:
self.logger.info(
"Checking if port {port} on {host} is in use".format(
port=port, host=host_server))
if ':' in port:
ports = port.split(':')
# Only check if the host port is free. The container port should be free
while self.check_port_used(host_server, ports[0]):
ports[0] = int(ports[0]) + 1
# Add this to the validated port list
validated_ports.append(
'{host_port}:{container_port}'.format(
host_port=str(ports[0]),
container_port=str(ports[1])))
else:
while self.check_port_used(host_server, port):
port = int(port) + 1
validated_ports.append(str(port))
self.logger.info(
'Adding app to formation {formation_name}: {hostname} cpu_shares={cpu} '
'ram={ram} ports={ports} host_server={host_server} docker_image={docker_image}'
.format(formation_name=formation_name,
hostname=hostname,
cpu=cpu_shares,
ram=ram,
ports=validated_ports,
host_server=host_server,
docker_image=docker_image))
f.add_app(None, '{hostname}'.format(hostname=hostname), cpu_shares,
ram, validated_ports, ssh_host_port, ssh_container_port,
host_server, docker_image, volume_list)
# Lets get this party started
for app in f.application_list:
self.start_application(app)
#self.logger.info("Sleeping 2 seconds while the container starts")
#time.sleep(2)
#self.bootstrap_application(app)
self.logger.info("Saving the formation to ETCD")
self.save_formation_to_etcd(f)
'Nebraska', 'Oklahoma', 'South Dakota', 'Texas', 'Washington', 'Other States' ]
# List of the sections of data that need to be found
name_labels = [ 'Cattle on Feed', 'Cattle Placed on Feed', 'Marketed', 'Other Disappearance' ]
end = site_contents.find('Number of Cattle on Feed on 1,000+ Capacity Feedlots by Month') # set to point at beginning of report (will be changed each iteration of following loop)
new_date = date - relativedelta(months = 1) # subtract one month because data is for previous month that report is published
n = 0
while n < len(name_labels):
end = site_contents.find(name_labels[n], end) # store where name_label occurs (always after previous name_label)
x = 0
while x < len(state_labels):
start = site_contents.find(state_labels[x], end) # find where the state name occurs
end = site_contents.find('\r\n', start) # end is changed to end of line
# This is the grammar for each line of data. It starts with the name of the state and is followed by a varying number
# of periods. Then five numbers of data follow the colon.
line_grammar = Literal(state_labels[x]) + Suppress(ZeroOrMore(Literal('.'))) + Suppress(Literal(':')) + Word(nums+',') * 5
parsed = line_grammar.parseString(site_contents[start:end])[3] # parse the line and only keep the fourth element because it contains most recent data
headings = ['Date', 'Thousand Head']
# The 'Cattle on Feed' data corresponds to the current month so 1 month is added to the date and
# the year, month, and day are converted to strings
if n == 0:
month = str((new_date + relativedelta(months = 1)).month)
day = (new_date + relativedelta(months = 1)).day
year = (new_date + relativedelta(months = 1)).year
else:
year = str(new_date.year)
month = str(new_date.month)
day = str(new_date.day)
if len(month) == 1:
month = '0' + month # prepend 0 to month if it is one digit
data = {'Date': [str(year) + str(month) + str(day)], 'Thousand Head': [parsed]}
data_df = pd.DataFrame(data, columns = headings)
from pyparsing import Literal, Empty, replaceWith
def insertResult(v):
"""
Parser helper function that simply inserts a result in
the list of values returned.
"""
return Empty().setParseAction(replaceWith(v))
p1 = Literal("1")
p2 = Literal("2") + insertResult(
"B") # 'AttributeError: 'NoneType' object has no attribute 'streamline''
p3 = insertResult("B") + Literal("3") # Blows python stack
r1 = p1.parseString("1")
r2 = p2.parseString("2")
r3 = p3.parseString("3")
print r2
print r3
def start_verifying(self):
# Parse out the username and formation name
# from the ETCD directory string
formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
# call out to ETCD and load all the formations
formation_list = []
user_list = self.etcd.list_directory('formations')
if user_list:
for user in user_list:
formations = self.etcd.list_directory(user)
for formation in formations:
parse_results = formation_parser.parseString(formation)
if parse_results:
formation_name = parse_results['formation_name']
username = parse_results['username']
self.logger.info('Attempting to load formation: {formation_name} '
'with username: {username}'.format(formation_name=formation_name,
username=username))
f = self.manager.load_formation_from_etcd(username, formation_name)
formation_list.append(f)
else:
self.logger.error("Could not parse the ETCD string")
if formation_list:
# TODO Use background salt jobs
# Start verifying things
# Ask salt to do these things for me and give me back an job_id
# results = self.salt_client.cmd_async(host, 'cmd.run',
# ['netstat -an | grep %s | grep tcp | grep -i listen' % port],
# expr_form='list')
#
# salt-run jobs.lookup_jid <job id number>
for f in formation_list:
for app in f.application_list:
# Check to make sure it's up and running
self.logger.info("Running verification on app: "
"{app_name}".format(app_name=app.hostname))
self.logger.info('{server} docker ps | grep {container_id}'.format(
server=app.host_server,
container_id=app.container_id))
results = self.salt_client.cmd(app.host_server, 'cmd.run',
['docker ps | grep {container_id}'.format(container_id=app.container_id)],
expr_form='list')
if results:
self.logger.debug("Salt return: {docker_results}".format(
docker_results=results[app.host_server]))
if results[app.host_server] == "":
self.logger.error("App {app} is not running!".format(
app=app.hostname))
# Start the app back up and run start.sh on there
self.start_application(app)
else:
self.logger.info("App {app} is running. Checking if "
"cron is running also".format(app=app.hostname))
# Check if cron is running on the container and bring it back
# up if needed
# Log in with ssh and check if cron is up and running
self.logger.info("Sleeping 2 seconds while the container starts")
time.sleep(2)
self.check_running_application(app)
else:
self.logger.error("Call out to server {server} failed. Moving it".format(
server=app.host_server))
# move the container
self.move_application(app)
def start_verifying(self):
# Parse out the username and formation name
# from the ETCD directory string
formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
# call out to ETCD and load all the formations
formation_list = []
user_list = self.etcd.list_directory('formations')
if user_list:
for user in user_list:
formations = self.etcd.list_directory(user)
for formation in formations:
parse_results = formation_parser.parseString(formation)
if parse_results:
formation_name = parse_results['formation_name']
username = parse_results['username']
self.logger.info(
'Attempting to load formation: {formation_name} '
'with username: {username}'.format(
formation_name=formation_name,
username=username))
f = self.manager.load_formation_from_etcd(
username, formation_name)
formation_list.append(f)
else:
self.logger.error("Could not parse the ETCD string")
if formation_list:
# TODO Use background salt jobs
# Start verifying things
# Ask salt to do these things for me and give me back an job_id
# results = self.salt_client.cmd_async(host, 'cmd.run',
# ['netstat -an | grep %s | grep tcp | grep -i listen' % port],
# expr_form='list')
#
# salt-run jobs.lookup_jid <job id number>
for f in formation_list:
for app in f.application_list:
# Check to make sure it's up and running
self.logger.info(
"Running verification on app: "
"{app_name}".format(app_name=app.hostname))
self.logger.info(
'{server} docker ps | grep {container_id}'.format(
server=app.host_server,
container_id=app.container_id))
results = self.salt_client.cmd(
app.host_server,
'cmd.run', [
'docker ps | grep {container_id}'.format(
container_id=app.container_id)
],
expr_form='list')
if results:
self.logger.debug(
"Salt return: {docker_results}".format(
docker_results=results[app.host_server]))
if results[app.host_server] == "":
self.logger.error(
"App {app} is not running!".format(
app=app.hostname))
# Start the app back up and run start.sh on there
self.start_application(app)
else:
self.logger.info(
"App {app} is running. Checking if "
"cron is running also".format(
app=app.hostname))
# Check if cron is running on the container and bring it back
# up if needed
# Log in with ssh and check if cron is up and running
self.logger.info(
"Sleeping 2 seconds while the container starts"
)
time.sleep(2)
self.check_running_application(app)
else:
self.logger.error(
"Call out to server {server} failed. Moving it"
.format(server=app.host_server))
# move the container
self.move_application(app)
try:
date=datetime.datetime.strptime(site_contents[begin_date:end_date].strip().replace(',',''), '%b %d %Y')
except ValueError:
date=datetime.datetime.strptime(site_contents[begin_date:end_date].strip().replace(',',''), '%B %d %Y')
date = date - datetime.timedelta(days = 5)
# list of each region in the report
labels = [ 'North East', 'South Atlantic', 'North Central', 'South Central', 'West', 'U.S. total' ]
# Loops through each region and uses pyparsing to find the head and average
# live weight for the turkeys slaughtered.
x = 0
while x < len(labels):
suppress = Suppress(Word(printables))
line = Literal(labels[x]) + suppress * 4 + Word(nums+',') + Word(nums+'.') # grammar for each line of data following a region
first = site_contents.find(labels[x]) # index of label
end = site_contents.find('\r\n', first) # index of end of the line
line = line.parseString(site_contents[first:end]) # parse line and store in list "line"
line = [float(y.replace(',','')) for y in line[1:]] # remove commas and convert to floats
headings = [ 'Date','Actual Turkey Slaughter', 'Turkey Average Weight' ]
data={ 'Date':[date.strftime('%Y-%m-%d')], 'Actual Turkey Slaughter': [line[0]], 'Turkey Average Weight': [line[1]] }
data_df = pd.DataFrame(data, columns = headings)
data_df.index = data_df['Date']
data_df = data_df.drop('Date', 1)
name = labels[x].replace(' ','_').replace('.','')
quandl_code = 'USDA_NW_PY021_' + name.upper() + '\r'
print 'code: ' + quandl_code
print 'name: Weekly National Turkey Slaughter- ' + labels[x].title() + '\r'
reference_text = ' Historical figures from USDA can be verified using the LMR datamart located ' \
'\n at http://mpr.datamart.ams.usda.gov.\n'
print 'description: Weekly national turkey slaughter data' \
'\n from the USDA NW_PY021 report published by the USDA Agricultural Marketing Service ' \
'\n (AMS). This dataset covers the ' + labels[x] + '.\n'\
class Manager(object):
'''
A manager to orchestrate the creation and
deletion of container clusters
'''
def __init__(self, logger):
self.salt_client = salt.client.LocalClient()
self.etcd = Etcd(logger)
self.logger = logger
# Parse out the username and formation name
# from the ETCD directory string
self.formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
def fqdn_to_shortname(self, fqdn):
if '.' in fqdn:
return fqdn.split('.')[0]
else:
return fqdn
def check_salt_key_used(self, hostname):
self.logger.info("Checking if the key for {host} is already used".format(
host=hostname))
s = subprocess.Popen('salt-key', shell=True, stdout=PIPE)
salt_list = s.communicate()[0]
if hostname in salt_list:
return True
else:
return False
def check_port_used(self, host, port):
self.logger.info("Checking if {port} on {host} is open with salt-client".format(
host=host, port=port))
results = self.salt_client.cmd(host, 'cmd.run',
['netstat -an | grep %s | grep tcp | grep -i listen' % port],
expr_form='list')
self.logger.debug("Salt return: {lsof}".format(lsof=results[host]))
if results[host] is not '':
return True
else:
return False
# TODO
def check_for_existing_formation(self, formation_name):
# If the user passed in an existing formation name lets append to it
pass
def get_docker_cluster(self):
# Return a list of docker hosts
cluster = self.etcd.get_key('docker_cluster')
if cluster is not None:
return cluster.split(',')
else:
return None
def get_load_balancer_cluster(self):
# Return a list of nginx hosts
cluster = self.etcd.get_key('nginx_cluster')
if cluster is not None:
return cluster.split(',')
else:
return None
def order_cluster_by_load(self, cluster_list):
# Sample salt output
# {'dlceph01.drwg.local': '0.27 0.16 0.15 1/1200 26234'}
# define grammar
point = Literal('.')
number = Word(nums)
floatnumber = Combine( number + point + number)
float_list = OneOrMore(floatnumber)
results = self.salt_client.cmd(','.join(cluster_list), 'cmd.run', ['cat /proc/loadavg'], expr_form='list')
load_list = []
self.logger.debug("Salt load return: {load}".format(load=results))
for host in results:
host_load = results[host]
match = float_list.parseString(host_load)
if match:
one_min = match[0]
five_min = match[1]
fifteen_min = match[2]
self.logger.debug("Adding Load({host}, {one_min}, {five_min}, {fifteen_min}".format(
host=host, one_min=one_min, five_min=five_min, fifteen_min=fifteen_min))
load_list.append(Load(host, one_min, five_min, fifteen_min))
else:
self.logger.error("Could not parse host load output")
# Sort the list by fifteen min load
load_list = sorted(load_list, key=lambda x: x.fifteen_min_load)
for load in load_list:
self.logger.debug("Sorted load list: " + str(load))
return load_list
# Retun a list of formations the user owns
def list_formations(self, username):
formation_list = []
formations = self.etcd.list_directory('formations/'+username)
for formation in formations:
parse_results = self.formation_parser.parseString(formation)
if parse_results:
formation_name = parse_results['formation_name']
formation_list.append(formation_name)
else:
self.logger.error("Could not parse the ETCD string")
self.logger.info('Formation list {formations} for user {user}'.format(
formations=formation_list, user=username))
return formation_list
# Load the formation and return a Formation object
def load_formation_from_etcd(self, username, formation_name):
f = Formation(username,formation_name)
app_list = json.loads(json.loads(
self.etcd.get_key('/formations/{username}/{formation_name}'.format(
username=username, formation_name=formation_name))))
for app in app_list:
# If our host doesn't support swapping we're going to get some garbage
# message in here
if "WARNING" in app['container_id']:
app['container_id'] = app['container_id'].replace("WARNING: Your "\
"kernel does not support memory swap capabilities. Limitation discarded.\n","")
#Message changed in docker 0.8.0
app['container_id'] = app['container_id'].replace("WARNING: WARNING:"\
"Your kernel does not support swap limit capabilities. Limitation "\
"discarded.\n","")
app['container_id'].strip('\n')
# Set volumes if needed
volumes = None
if app['volumes']:
self.logger.info("Setting volumes to: " + ''.join(app['volumes']))
volumes = app['volumes']
f.add_app(app['container_id'], app['hostname'], app['cpu_shares'],
app['ram'], app['port_list'], app['ssh_port'], 22, app['host_server'], volumes)
# Return fully parsed and populated formation object
return f
def save_formation_to_etcd(self, formation):
name = formation.name
username = formation.username
self.etcd.set_key('formations/{username}/{formation_name}'.format(
username=username, formation_name=name), formation)
# TODO write code to add new apps to load balancer
def add_app_to_nginx(self, app):
pass
# TODO write code to add new apps to the load balancer
def add_app_to_apache(self, app):
pass
def start_application(self, app):
# Run a salt cmd to startup the formation
docker_command = "docker run -c={cpu_shares} -d -i -t -h=\"{hostname}\" -m={ram}m "\
"--name={hostname} {port_list} {volume_list} {image} /sbin/my_init -- bash"
self.logger.info("Port list %s" % app.port_list)
port_list = ' '.join(map(lambda x: '-p ' + x, app.port_list))
# Only create this list if needed
volume_list = ''
if app.volume_list:
volume_list = ' '.join(map(lambda x: '-v ' + x, app.volume_list))
d = docker_command.format(cpu_shares=app.cpu_shares,
hostname=app.hostname, ram=app.ram, image=app.docker_image,
port_list=port_list, volume_list=volume_list)
self.logger.info("Starting up docker container on {host_server} with cmd: {docker_cmd}".format(
host_server=app.host_server, docker_cmd=d))
salt_process = self.salt_client.cmd(app.host_server,'cmd.run', [d], expr_form='list')
container_id = salt_process[app.host_server]
if container_id:
if "WARNING" in container_id:
container_id = container_id.replace("WARNING: "\
"Your kernel does not support swap limit capabilities. Limitation "\
"discarded.\n","")
container_id.strip("\n")
#Docker only uses the first 12 chars to identify a container
app.change_container_id(container_id[0:12])
def bootstrap_application(self, app):
# Log into the host with paramiko and run the salt bootstrap script
host_server = self.fqdn_to_shortname(app.host_server)
self.logger.info("Bootstrapping {hostname} on server: {host_server} port: {port}".format(
hostname=app.hostname,
host_server=host_server,
port=app.ssh_port))
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=host_server, port=app.ssh_port,
username='******', password='******')
transport = paramiko.Transport((host_server, app.ssh_port))
transport.connect(username = '******', password = '******')
sftp = paramiko.SFTPClient.from_transport(transport)
sftp.put('bootstrap.sh', '/root/bootstrap.sh')
sftp.put('start.sh', '/root/start.sh')
ssh.exec_command("chmod +x /root/bootstrap.sh")
ssh.exec_command("chmod +x /root/start.sh")
stdin, stdout, stderr = ssh.exec_command("bash /root/start.sh")
self.logger.debug(''.join(stdout.readlines()))
ssh.close()
except SSHException:
self.logger.error("Failed to log into server. Shutting it down and cleaning up the mess.")
self.delete_container(app.host_server, app.container_id)
# Stops and deletes a container
def delete_container(self, host_server, container_id):
results = self.salt_client.cmd(host_server, 'cmd.run',
['docker stop {container_id}'.format(container_id=container_id)],
expr_form='list')
self.logger.debug("Salt return: {stop_cmd}".format(stop_cmd=results[host_server]))
results = self.salt_client.cmd(host_server, 'cmd.run',
['docker rm {container_id}'.format(container_id=container_id)],
expr_form='list')
self.logger.debug("Salt return: {rm_cmd}".format(rm_cmd=results[host_server]))
# Stops and deletes a formation. Use with caution
def delete_formation(self, user, formation_name):
formation_list = self.list_formations(user)
if formation_name in formation_list:
pass
else:
self.logger.error("Formation name not found!")
def list_containers(self, user, formation_name):
pass
def create_containers(self, user, number, formation_name,
cpu_shares, ram, port_list, hostname_scheme, volume_list,
docker_image, force_host_server=None):
f = Formation(user, formation_name)
# Convert ram to bytes from MB
ram = ram * 1024 * 1024
# Get the cluster machines on each creation
cluster_list = self.get_docker_cluster()
circular_cluster_list = CircularList(self.order_cluster_by_load(cluster_list))
# Loop for the requested amount of containers to be created
for i in range(1, number+1):
# [{"host_port":ssh_host_port, "container_port":ssh_container_port}]
ssh_host_port = 9022 + i
ssh_container_port = 22
host_server = circular_cluster_list[i].hostname
hostname = '{hostname}{number}'.format(
hostname=hostname_scheme,
number=str(i).zfill(3))
# First check if we can add this host to salt. If not exit with -1
if self.check_salt_key_used(hostname):
self.logger.error('Salt key is already taken for {hostname}'.format(
hostname=hostname))
sys.exit(-1)
# We are being asked to overwrite this
if force_host_server:
host_server = force_host_server
validated_ports = []
while self.check_port_used(host_server, ssh_host_port):
ssh_host_port = ssh_host_port +1
for port in port_list:
self.logger.info("Checking if port {port} on {host} is in use".format(
port=port, host=host_server))
if ':' in port:
ports = port.split(':')
# Only check if the host port is free. The container port should be free
while self.check_port_used(host_server, ports[0]):
ports[0] = int(ports[0]) + 1
# Add this to the validated port list
validated_ports.append('{host_port}:{container_port}'.format(
host_port = str(ports[0]),
container_port = str(ports[1])))
else:
while self.check_port_used(host_server, port):
port = int(port) + 1
validated_ports.append(str(port))
self.logger.info('Adding app to formation {formation_name}: {hostname} cpu_shares={cpu} '
'ram={ram} ports={ports} host_server={host_server} docker_image={docker_image}'.format(
formation_name=formation_name, hostname=hostname,
cpu=cpu_shares, ram=ram, ports=validated_ports, host_server=host_server,
docker_image=docker_image))
f.add_app(None, '{hostname}'.format(hostname=hostname),
cpu_shares, ram, validated_ports, ssh_host_port,
ssh_container_port, host_server, docker_image, volume_list)
# Lets get this party started
for app in f.application_list:
self.start_application(app)
#self.logger.info("Sleeping 2 seconds while the container starts")
#time.sleep(2)
#self.bootstrap_application(app)
self.logger.info("Saving the formation to ETCD")
self.save_formation_to_etcd(f)
# names of each cut in the report
labels = ['BREAST - B/S', 'TENDERLOINS', 'BREAST - WITH RIBS', 'BREAST - LINE RUN', 'LEGS', 'LEG QUARTERS (BULK)',\
'DRUMSTICKS', 'THIGHS', 'B/S THIGHS', 'WINGS (WHOLE)', 'BACKS AND NECKS (STRIPPED)', 'LIVERS (5 POUND TUBS)',\
'GIZZARDS (HEARTS)']
ending_index = 0 # initializes ending_index to 0 to be used in following loop
# Loops through each cut in labels and uses pyparsing to find the weighted average
# and volume for that cut. The data and data are formatted into a table and the
# relevant quandl data is printed.
x = 0
while x < len(labels):
line = Literal(labels[x]) + Word(nums+'-') + Word(nums+'.') + Word(nums+',') # grammar to find each label's data
starting_index = site_contents.find(labels[x], ending_index) # stores the index of the beginning of each label's data
ending_index = site_contents.find('\r\n', starting_index) # stores the index of the end of the label's data
text = site_contents[starting_index:ending_index] # the line to be parsed is from starting_index to ending_index
parsed = line.parseString(text) # parses the line and stores it in "parsed"
headings = ['Date', 'Weighted Average (Price)', 'Volume (Lbs)']
data = {'Date': [date.strftime('%Y-%m-%d')], 'Weighted Average (Price)': [parsed[2]], 'Volume (Lbs)': [parsed[3].replace(',','')]}
data_df = pd.DataFrame(data, columns = headings)
data_df.index = data_df['Date']
data_df = data_df.drop('Date', 1)
replace = re.compile('[ /]') # list of characters to be replaced
remove = re.compile('[,%#-&()!$+<>?/\'"{}.*@]') # list of characters to be removed
name1 = replace.sub('_', labels[x]) # replace certain characters with '_'
name2 = remove.sub('', name1).upper() # remove certain characters and convert to upper case
name2 = name2.translate(None, '-') # ensure '-' character is removed
quandl_code = 'USDA_AJ_PY047_' + name2 + '\r'
print 'code: ' + quandl_code
print 'name: Daily Northeast Broiler/Fryer Parts- ' + labels[x].title() + '\r'
reference_text = ' Historical figures from USDA can be verified using the LMR datamart located ' \
'\n at http://mpr.datamart.ams.usda.gov.\n'
