parser.add_option('-x',
'--destroy',
dest='destroy',
action='store_true',
default=False,
help='Whether to destroy the group without rebuilding')
parser.add_option('-y',
'--delete',
dest='destroy',
action='store_true',
default=False,
help='Whether to destroy the group without rebuilding')
(options, args) = parser.parse_args()
conn = connect_autoscale_to(options.region)
lcname = options.group + '_config'
groups = filter(lambda x: x.name == GROUP_NAME, conn.get_all_groups())
group = groups[0] if groups else None
def create_group():
global conn, lcname
lc = LaunchConfiguration(name=lcname,
image_id=options.ami,
key_name='relwellnlp',
instance_type='m2.4xlarge',
security_groups=['sshable'])
conn.create_launch_configuration(lc)
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
SIG = str(os.getpid()) + '_' + str(int(time()))
TEXT_DIR = '/tmp/text/'
XML_DIR = '/tmp/xml/'
PACKAGE_DIR = "/tmp/event_packages/"
BUCKET_NAME = 'nlp-data'
GROUP = 'parser_poller'
REGION = 'us-west-2'
DESIRED_CAPACITY = 4
conn = connect_s3()
bucket = conn.get_bucket(BUCKET_NAME)
hostname = gethostname()
autoscale = connect_autoscale_to(REGION)
autoscale_group = None
groups = autoscale.get_all_groups(names=[GROUP])
stalling_increments = 0
def add_files():
global hostname, bucket, PACKAGE_DIR, SIG, inqueue, autoscale_group
print "[%s] Adding to text queue" % hostname
keys = filter(lambda x:x.key.endswith('.tgz'), bucket.list('text_events'))
# iterating over keys in case we try to grab a key that another instance scoops
for key in keys:
old_key_name = key.key
print "[%s] found key %s" % (hostname, old_key_name)
# found a tar file, now try to capture it via move
parser.add_option('-r', '--region', dest='region', action='store', default='us-west-2',
help='Amazon region to connect to')
parser.add_option('-z', '--zones', dest='zones', action='store', default='us-west-2',
help="Availability zones for this autoscale group")
parser.add_option('-b', '--rebuild', dest='rebuild', action='store_true', default=False,
help='Whether to rebuild the group (deletes the old group)')
parser.add_option('-c', '--create', dest='create', action='store_true', default=False,
help='Whether to create the group for the first time')
parser.add_option('-x', '--destroy', dest='destroy', action='store_true', default=False,
help='Whether to destroy the group without rebuilding')
parser.add_option('-y', '--delete', dest='destroy', action='store_true', default=False,
help='Whether to destroy the group without rebuilding')
(options, args) = parser.parse_args()
conn = connect_autoscale_to(options.region)
lcname = options.group+'_config'
groups = filter(lambda x:x.name == GROUP_NAME, conn.get_all_groups())
group = groups[0] if groups else None
def create_group():
global conn, lcname
lc = LaunchConfiguration(name=lcname,
image_id=options.ami,
key_name='relwellnlp',
instance_type='m2.4xlarge',
security_groups=['sshable'])
conn.create_launch_configuration(lc)
