Пример #1
0
def save(data, stats):
    res=Mep.get_by_id(data['UserID'])
    if res is not None:
        if 'Gender' not in data and 'Gender' in res.data: data['Gender']=res['Gender']
        d=diff(dict([(k,v) for k,v in res.data.items() if not k in ['meta', 'changes', 'activities',]]),
               dict([(k,v) for k,v in data.items() if not k in ['meta', 'changes', 'activities',]]))
        data['changes']=res.data.get('changes',{})
    else:
        d=diff({}, dict([(k,v) for k,v in data.items() if not k in ['meta', 'changes', 'activities',]]))
        data['changes']={}
    if d:
        now=datetime.utcnow().replace(microsecond=0)
        if not res:
            logger.info('adding %s' % (data['Name']['full']))
            data['meta']['created']=now
            if stats: stats[0]+=1
            data['changes']={}
        else:
            logger.info('updating %s' % (data['Name']['full']))
            logger.warn(jdump(d))
            data['meta']['updated']=now
            if stats: stats[1]+=1
            data['id']=res.id
            data['changes']=res.data.get('changes',{})
        data['changes'][now.isoformat()]=d
        Mep.upsert(data)
    del res
    if stats:
        del data
        return stats
    else: return data
Пример #2
0
  def attach_disk(self, parameters, disk_name, instance_id):
    """ Attaches the Elastic Block Store (EBS) volume specified in 'disk_name'
    to this virtual machine.

    This method differs from its EC2 counterpart because in EC2, we can ask the
    cloud to attach the disk to a certain location. In Euca, it determines where
    the disk gets placed, so we have to learn where it placed the disk and
    return that location instead.

    Args:
      parameters: A dict with keys for each parameter needed to connect to AWS.
      disk_name: A str naming the EBS volume to attach to this machine.
      instance_id: A str naming the id of the instance that the disk should be
        attached to. In practice, callers add disks to their own instances.
    Returns:
      The location on the local filesystem where the disk has been attached.
    """
    devices_before_attach = glob.glob('/dev/*')
    EC2Agent.attach_disk(self, parameters, disk_name, instance_id)
    while True:
      devices_after_attach = glob.glob('/dev/*')
      new_devices = utils.diff(devices_after_attach, devices_before_attach)
      if new_devices:
        utils.log("Found new attached devices: {0}".format(new_devices))
        if len(new_devices) == 1:
          utils.log("Found exactly one new attached device at {0}".format(
            new_devices[0]))
          return new_devices[0]
        else:
          self.handle_failure("Found too many new attached devices - not sure" \
            " which one is the device we attached. New devices are {0}".format(
            new_devices))
      else:
        utils.log("Still waiting for attached device to appear.")
        utils.sleep(1)
Пример #3
0
  def run_instances(self, count, parameters, security_configured):
    """ Starts 'count' instances in Microsoft Azure, and returns once they
    have been started. Callers should create a network and attach a firewall
    to it before using this method, or the newly created instances will not
    have a network and firewall to attach to (and thus this method will fail).
    Args:
      count: An int, that specifies how many virtual machines should be started.
      parameters: A dict, containing all the parameters necessary to
        authenticate this user with Azure.
      security_configured: Unused, as we assume that the network and firewall
        has already been set up.
    Returns:
      instance_ids: A list of unique Azure VM names.
      public_ips: A list of public IP addresses.
      private_ips: A list of private IP addresses.
    """
    credentials = self.open_connection(parameters)
    subscription_id = str(parameters[self.PARAM_SUBSCRIBER_ID])
    group_name = parameters[self.PARAM_RESOURCE_GROUP]
    network_client = NetworkManagementClient(credentials, subscription_id)
    active_public_ips, active_private_ips, active_instances = \
      self.describe_instances(parameters)
    virtual_network = parameters[self.PARAM_GROUP]
    subnet = network_client.subnets.get(group_name, virtual_network,
                                        virtual_network)
    threads = []
    for _ in range(count):
      vm_network_name = Haikunator().haikunate()
      thread = threading.Thread(target=self.setup_network_and_create_vm,
                                args=(network_client, parameters, subnet,
                                      vm_network_name))
      thread.start()
      threads.append(thread)

    for x in threads:
      x.join()

    public_ips, private_ips, instance_ids = self.describe_instances(parameters)
    public_ips = utils.diff(public_ips, active_public_ips)
    private_ips = utils.diff(private_ips, active_private_ips)
    instance_ids = utils.diff(instance_ids, active_instances)
    return instance_ids, public_ips, private_ips
Пример #4
0
def save(data, stats):
    if not data: return stats
    res=Dossier.get_by_src(data['meta']['source'])
    if res is not None:
        d=diff(dict([(k,v) for k,v in res.items() if not k in ['meta', 'changes']]),
               dict([(k,v) for k,v in data.items() if not k in ['meta', 'changes',]]))
        data['changes']=res.data.get('changes',{})
    else:
        d=diff({}, dict([(k,v) for k,v in data.items() if not k in ['meta', 'changes',]]))
        data['changes']={}
    #logger.warn(pprint.pformat(d))
    if d:
        now=data['meta']['timestamp'].replace(microsecond=0).isoformat()
        del data['meta']['timestamp']
        if not res:
            logger.info(('adding %s - %s' % (data['procedure']['reference'],data['procedure']['title'])).encode('utf8'))
            data['meta']['created']=now
            stats[0]+=1
        else:
            logger.info(('updating  %s - %s' % (data['procedure']['reference'],data['procedure']['title'])).encode('utf8'))
            data['meta']['updated']=now
            stats[1]+=1
            logger.info(jdump(d))
        #if not NOMAIL:
        #    m=db.notifications.find({'dossiers': data['procedure']['reference']},['active_emails'])
        #    for g in m:
        #        if len(g['active_emails'])==0:
        #            continue
        #        msg = Message("[PT] %s %s" % (data['procedure']['reference'],data['procedure']['title']),
        #                      sender = "*****@*****.**",
        #                      bcc = g['active_emails'])
        #        #msg.html = htmldiff(data,d)
        #        msg.body = makemsg(data,d)
        #        mail.send(msg)
        #logger.info(htmldiff(data,d))
        #logger.info(makemsg(data,d))
        data['changes'][now]=d
        Dossier.upsert(data)
    return stats
Пример #5
0
    def attach_disk(self, parameters, disk_name, instance_id):
        """ Attaches the Elastic Block Store (EBS) volume specified in 'disk_name'
    to this virtual machine.

    This method differs from its EC2 counterpart because in EC2, we can ask the
    cloud to attach the disk to a certain location. In Euca, it determines where
    the disk gets placed, so we have to learn where it placed the disk and
    return that location instead.

    Args:
      parameters: A dict with keys for each parameter needed to connect to AWS.
      disk_name: A str naming the EBS volume to attach to this machine.
      instance_id: A str naming the id of the instance that the disk should be
        attached to. In practice, callers add disks to their own instances.
    Returns:
      The location on the local filesystem where the disk has been attached.
    """
        devices_before_attach = glob.glob('/dev/*')
        EC2Agent.attach_disk(self, parameters, disk_name, instance_id)
        while True:
            devices_after_attach = glob.glob('/dev/*')
            new_devices = utils.diff(devices_after_attach,
                                     devices_before_attach)
            if new_devices:
                utils.log(
                    "Found new attached devices: {0}".format(new_devices))
                if len(new_devices) == 1:
                    utils.log(
                        "Found exactly one new attached device at {0}".format(
                            new_devices[0]))
                    return new_devices[0]
                else:
                    self.handle_failure("Found too many new attached devices - not sure" \
                      " which one is the device we attached. New devices are {0}".format(
                      new_devices))
            else:
                utils.log("Still waiting for attached device to appear.")
                utils.sleep(1)
Пример #6
0
  def run_instances(self, count, parameters, security_configured):
    """
    Spawns the specified number of EC2 instances using the parameters
    provided. This method is blocking in that it waits until the
    requested VMs are properly booted up. However if the requested
    VMs cannot be procured within 1800 seconds, this method will treat
    it as an error and return. (Also see documentation for the BaseAgent
    class)

    Args:
      count               No. of VMs to spawned
      parameters          A dictionary of parameters. This must contain 'keyname',
                          'group', 'image_id' and 'instance_type' parameters.
      security_configured Uses this boolean value as an heuristic to
                          detect brand new AppScale deployments.

    Returns:
      A tuple of the form (instances, public_ips, private_ips)
    """
    image_id = parameters[self.PARAM_IMAGE_ID]
    instance_type = parameters[self.PARAM_INSTANCE_TYPE]
    keyname = parameters[self.PARAM_KEYNAME]
    group = parameters[self.PARAM_GROUP]
    spot = parameters[self.PARAM_SPOT]

    utils.log('[{0}] [{1}] [{2}] [{3}] [ec2] [{4}] [{5}]'.format(count,
      image_id, instance_type, keyname, group, spot))


    credentials = parameters[self.PARAM_CREDENTIALS]
    creds = parameters['credentials']
    f = open('userfile','w')
    userstr  = """#!/bin/bash \nset -x\nexec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\ntouch anand3.txt\necho "testing logfile"\necho BEGIN\ndate '+%Y-%m-%d %H:%M:%S'\necho END\ntouch anand2.txt\n"""
    userstr+='export AWS_ACCESS_KEY_ID={0}\n'.format(str(credentials['EC2_ACCESS_KEY']))
    userstr+='export AWS_SECRET_ACCESS_KEY={0}\n'.format( str(credentials['EC2_SECRET_KEY']))
    userstr+='echo export AWS_ACCESS_KEY_ID={0} >> ~/.bashrc\n'.format(str(credentials['EC2_ACCESS_KEY']))
    userstr+='echo export AWS_SECRET_ACCESS_KEY={0} >> ~/.bashrc\n'.format( str(credentials['EC2_SECRET_KEY']))
    userstr+='echo export AWS_ACCESS_KEY_ID={0} >> /home/ubuntu/.bashrc\n'.format(str(credentials['EC2_ACCESS_KEY']))   
    userstr+='echo export AWS_SECRET_ACCESS_KEY={0} >> /home/ubuntu/.bashrc\n'.format( str(credentials['EC2_SECRET_KEY']))

    userstr+='export STOCHKIT_HOME={0}\n'.format('/home/ubuntu/StochKit/')
    userstr+='export STOCHKIT_ODE={0}\n'.format('/home/ubuntu/ode/')
    userstr+='echo export STOCHKIT_HOME={0} >> ~/.bashrc\n'.format("/home/ubuntu/StochKit/")
    userstr+='echo export STOCHKIT_HOME={0} >> /home/ubuntu/.bashrc\n'.format("/home/ubuntu/StochKit/")
    userstr+='echo export STOCHKIT_ODE={0} >> ~/.bashrc\n'.format("/home/ubuntu/ode/")
    userstr+='echo export STOCHKIT_ODE={0} >> /home/ubuntu/.bashrc\n'.format("/home/ubuntu/ode/")
    userstr+='source ~/.bashrc \n'
    userstr+='source /home/ubuntu/.bashrc \n'
    # Workers need an alarm...
    skip_alarm = False
    if self.PARAM_QUEUE_HEAD in parameters and parameters[self.PARAM_QUEUE_HEAD]:
      # ...but the queue head doesnt
      skip_alarm = True
      # Queue head, needs to have at least two cores
      insufficient_cores = ['t1.micro', 'm1.small', 'm1.medium', 'm3.medium']
      if instance_type in insufficient_cores:
        instance_type = 'c3.large'
      # Create the user that we want to use to connect to the broker
      # and configure its permissions on the default vhost.
      userstr += "rabbitmqctl add_user stochss ucsb\n"
      userstr += 'rabbitmqctl set_permissions -p / stochss ".*" ".*" ".*"\n'
      # userstr += "rabbitmq-server -detached\n"
    else:
      # Update celery config file...it should have the correct IP
      # of the Queue head node, which should already be running.
      celery_config_filename = os.path.join(
        os.path.dirname(os.path.abspath(__file__)),
        "../celeryconfig.py"
      )
      # Pass it line by line so theres no weird formatting errors from 
      # trying to echo a multi-line file directly on the command line
      with open(celery_config_filename, 'r') as celery_config_file:
        lines = celery_config_file.readlines()
        # Make sure we overwrite the file with our first write
        userstr += "echo '{0}' > /home/ubuntu/celeryconfig.py\n".format(lines[0])
        for line in lines[1:]:
          userstr += "echo '{0}' >> /home/ubuntu/celeryconfig.py\n".format(line)
    # Even the queue head gets a celery worker
    # NOTE: We only need to use the -n argument to celery command if we are starting
    #       multiple workers on the same machine. Instead, we are starting one worker
    #       per machine and letting that one worker execute one task per core, using
    #       the configuration in celeryconfig.py to ensure that Celery detects the 
    #       number of cores and enforces this desired behavior.
    if self.PARAM_WORKER_QUEUE in parameters:
      userstr+="nohup celery -A tasks worker --autoreload --loglevel=info -Q {0} --workdir /home/ubuntu > /home/ubuntu/nohup.log 2>&1 & \n".format(
          parameters[self.PARAM_WORKER_QUEUE]
      )
    else:
      userstr+="nohup celery -A tasks worker --autoreload --loglevel=info --workdir /home/ubuntu > /home/ubuntu/nohup.log 2>&1 & \n"
    f.write(userstr)
    f.close()
    start_time = datetime.datetime.now()
    active_public_ips = []
    active_private_ips = []
    active_instances = []

    try:
      attempts = 1
      while True:
        instance_info = self.describe_instances_old(parameters)
        active_public_ips = instance_info[0]
        active_private_ips = instance_info[1]
        active_instances = instance_info[2]

        # If security has been configured on this agent just now,
        # that's an indication that this is a fresh cloud deployment.
        # As such it's not expected to have any running VMs.
        if len(active_instances) > 0 or security_configured:
          break
        elif attempts == self.DESCRIBE_INSTANCES_RETRY_COUNT:
          self.handle_failure('Failed to invoke describe_instances')
        attempts += 1

      conn = self.open_connection(parameters)
      if spot == 'True':
        price = parameters[self.PARAM_SPOT_PRICE]
        conn.request_spot_instances(str(price), image_id, key_name=keyname,
          security_groups=[group], instance_type=instance_type, count=count, user_data = userstr)
      else:
        conn.run_instances(image_id, count, count, key_name=keyname,
          security_groups=[group], instance_type=instance_type, user_data=userstr)

      instance_ids = []
      public_ips = []
      private_ips = []
      utils.sleep(10)
      end_time = datetime.datetime.now() + datetime.timedelta(0,
        self.MAX_VM_CREATION_TIME)
      now = datetime.datetime.now()

      while now < end_time:
        time_left = (end_time - now).seconds
        utils.log('[{0}] {1} seconds left...'.format(now, time_left))
        instance_info = self.describe_instances_old(parameters)
        public_ips = instance_info[0]
        private_ips = instance_info[1]
        instance_ids = instance_info[2]
        public_ips = utils.diff(public_ips, active_public_ips)
        private_ips = utils.diff(private_ips, active_private_ips)
        instance_ids = utils.diff(instance_ids, active_instances)
        if count == len(public_ips):
          break
        time.sleep(self.SLEEP_TIME)
        now = datetime.datetime.now()

      if not public_ips:
        self.handle_failure('No public IPs were able to be procured '
                            'within the time limit')

      if len(public_ips) != count:
        for index in range(0, len(public_ips)):
          if public_ips[index] == '0.0.0.0':
            instance_to_term = instance_ids[index]
            utils.log('Instance {0} failed to get a public IP address and' \
                      ' is being terminated'.format(instance_to_term))
            conn.terminate_instances([instance_to_term])

      end_time = datetime.datetime.now()
      total_time = end_time - start_time
      if spot:
        utils.log('TIMING: It took {0} seconds to spawn {1} spot ' \
                  'instances'.format(total_time.seconds, count))
      else:
        utils.log('TIMING: It took {0} seconds to spawn {1} ' \
                  'regular instances'.format(total_time.seconds, count))
        if not skip_alarm:
          utils.log('Creating Alarms for the instances')
          for machineid in instance_ids:
              self.make_sleepy(parameters, machineid)   
      return instance_ids, public_ips, private_ips
    except EC2ResponseError as exception:
      self.handle_failure('EC2 response error while starting VMs: ' +
                          exception.error_message)
    except Exception as exception:
      if isinstance(exception, AgentRuntimeException):
        raise exception
      else:
        self.handle_failure('Error while starting VMs: ' + exception.message)
Пример #7
0
  def run_instances(self, count, parameters, security_configured):
    """
    Spawns the specified number of EC2 instances using the parameters
    provided. This method is blocking in that it waits until the
    requested VMs are properly booted up. However if the requested
    VMs cannot be procured within 1800 seconds, this method will treat
    it as an error and return. (Also see documentation for the BaseAgent
    class)

    Args:
      count               No. of VMs to spawned
      parameters          A dictionary of parameters. This must contain 'keyname',
                          'group', 'image_id' and 'instance_type' parameters.
      security_configured Uses this boolean value as an heuristic to
                          detect brand new AppScale deployments.

    Returns:
      A tuple of the form (instances, public_ips, private_ips)
    """
    image_id = parameters[self.PARAM_IMAGE_ID]
    instance_type = parameters[self.PARAM_INSTANCE_TYPE]
    keyname = parameters[self.PARAM_KEYNAME]
    group = parameters[self.PARAM_GROUP]
    spot = parameters[self.PARAM_SPOT]

    utils.log('[{0}] [{1}] [{2}] [{3}] [ec2] [{4}] [{5}]'.format(count,
      image_id, instance_type, keyname, group, spot))

    start_time = datetime.datetime.now()
    active_public_ips = []
    active_private_ips = []
    active_instances = []

    try:
      attempts = 1
      while True:
        instance_info = self.describe_instances(parameters)
        active_public_ips = instance_info[0]
        active_private_ips = instance_info[1]
        active_instances = instance_info[2]

        # If security has been configured on this agent just now,
        # that's an indication that this is a fresh cloud deployment.
        # As such it's not expected to have any running VMs.
        if len(active_instances) > 0 or security_configured:
          break
        elif attempts == self.DESCRIBE_INSTANCES_RETRY_COUNT:
          self.handle_failure('Failed to invoke describe_instances')
        attempts += 1

      conn = self.open_connection(parameters)
      if spot == 'True':
        price = parameters[self.PARAM_SPOT_PRICE]
        conn.request_spot_instances(str(price), image_id, key_name=keyname,
          security_groups=[group], instance_type=instance_type, count=count)
      else:
        conn.run_instances(image_id, count, count, key_name=keyname,
          security_groups=[group], instance_type=instance_type)

      instance_ids = []
      public_ips = []
      private_ips = []
      utils.sleep(10)
      end_time = datetime.datetime.now() + datetime.timedelta(0,
        self.MAX_VM_CREATION_TIME)
      now = datetime.datetime.now()

      while now < end_time:
        time_left = (end_time - now).seconds
        utils.log('[{0}] {1} seconds left...'.format(now, time_left))
        instance_info = self.describe_instances(parameters)
        public_ips = instance_info[0]
        private_ips = instance_info[1]
        instance_ids = instance_info[2]
        public_ips = utils.diff(public_ips, active_public_ips)
        private_ips = utils.diff(private_ips, active_private_ips)
        instance_ids = utils.diff(instance_ids, active_instances)
        if count == len(public_ips):
          break
        time.sleep(self.SLEEP_TIME)
        now = datetime.datetime.now()

      if not public_ips:
        self.handle_failure('No public IPs were able to be procured '
                            'within the time limit')

      if len(public_ips) != count:
        for index in range(0, len(public_ips)):
          if public_ips[index] == '0.0.0.0':
            instance_to_term = instance_ids[index]
            utils.log('Instance {0} failed to get a public IP address and' \
                      ' is being terminated'.format(instance_to_term))
            conn.terminate_instances([instance_to_term])

      end_time = datetime.datetime.now()
      total_time = end_time - start_time
      if spot:
        utils.log('TIMING: It took {0} seconds to spawn {1} spot ' \
                  'instances'.format(total_time.seconds, count))
      else:
        utils.log('TIMING: It took {0} seconds to spawn {1} ' \
                  'regular instances'.format(total_time.seconds, count))
      return instance_ids, public_ips, private_ips
    except EC2ResponseError as exception:
      self.handle_failure('EC2 response error while starting VMs: ' +
                          exception.error_message)
    except Exception as exception:
      if isinstance(exception, AgentRuntimeException):
        raise exception
      else:
        self.handle_failure('Error while starting VMs: ' + exception.message)
Пример #8
0
  def run_instances(self, count, parameters, security_configured):
    """ Starts 'count' instances in Google Compute Engine, and returns once they
    have been started.

    Callers should create a network and attach a firewall to it before using
    this method, or the newly created instances will not have a network and
    firewall to attach to (and thus this method will fail).

    Args:
      count: An int that specifies how many virtual machines should be started.
      parameters: A dict with keys for each parameter needed to connect to
        Google Compute Engine.
      security_configured: Unused, as we assume that the network and firewall
        has already been set up.
    """
    project_id = parameters[self.PARAM_PROJECT]
    image_id = parameters[self.PARAM_IMAGE_ID]
    instance_type = parameters[self.PARAM_INSTANCE_TYPE]
    keyname = parameters[self.PARAM_KEYNAME]
    group = parameters[self.PARAM_GROUP]
    zone = parameters[self.PARAM_ZONE]

    utils.log("Starting {0} machines with machine id {1}, with " \
      "instance type {2}, keyname {3}, in security group {4}, in zone {5}" \
      .format(count, image_id, instance_type, keyname, group, zone))

    # First, see how many instances are running and what their info is.
    start_time = datetime.datetime.now()
    active_public_ips, active_private_ips, active_instances = \
      self.describe_instances(parameters)

    # Construct URLs
    image_url = '{0}{1}/global/images/{2}'.format(self.GCE_URL, project_id, image_id)
    project_url = '{0}{1}'.format(self.GCE_URL, project_id)
    machine_type_url = '{0}/zones/{1}/machineTypes/{2}'.format(project_url,
      zone, instance_type)
    zone_url = '{0}/zones/{1}'.format(project_url, zone)
    network_url = '{0}/global/networks/{1}'.format(project_url, group)

    # Construct the request body
    for index in range(count):
      disk_url = self.create_scratch_disk(parameters)
      instances = {
        # Truncate the name down to the first 62 characters, since GCE doesn't
        # let us use arbitrarily long instance names.
        'name': "appscale-{0}-{1}".format(group, uuid.uuid4())[:62],
        'machineType': machine_type_url,
       'disks':[{
         'source': disk_url,
         'boot': 'true',
         'type': 'PERSISTENT'
        }],
        'image': image_url,
        'networkInterfaces': [{
          'accessConfigs': [{
            'type': 'ONE_TO_ONE_NAT',
            'name': 'External NAT'
           }],
          'network': network_url
        }],
        'serviceAccounts': [{
             'email': self.DEFAULT_SERVICE_EMAIL,
             'scopes': [self.GCE_SCOPE]
        }]
      }

      # Create the instance
      gce_service, credentials = self.open_connection(parameters)
      http = httplib2.Http()
      auth_http = credentials.authorize(http)
      request = gce_service.instances().insert(
           project=project_id, body=instances, zone=zone)
      response = request.execute(auth_http)
      utils.log(str(response))
      self.ensure_operation_succeeds(gce_service, auth_http, response, 
        parameters[self.PARAM_PROJECT])
    
    instance_ids = []
    public_ips = []
    private_ips = []
    end_time = datetime.datetime.now() + datetime.timedelta(0,
      self.MAX_VM_CREATION_TIME)
    now = datetime.datetime.now()

    while now < end_time:
      time_left = (end_time - now).seconds
      utils.log("Waiting for your instances to start...")
      instance_info = self.describe_instances(parameters)
      public_ips = instance_info[0]
      private_ips = instance_info[1]
      instance_ids = instance_info[2]
      public_ips = utils.diff(public_ips, active_public_ips)
      private_ips = utils.diff(private_ips, active_private_ips)
      instance_ids = utils.diff(instance_ids, active_instances)
      if count == len(public_ips):
        break
      time.sleep(self.SLEEP_TIME)
      now = datetime.datetime.now()

    if not public_ips:
      self.handle_failure('No public IPs were able to be procured '
                          'within the time limit')

    if len(public_ips) != count:
      for index in range(0, len(public_ips)):
        if public_ips[index] == '0.0.0.0':
          instance_to_term = instance_ids[index]
          utils.log('Instance {0} failed to get a public IP address'\
                  'and is being terminated'.format(instance_to_term))
          self.terminate_instances([instance_to_term])

    end_time = datetime.datetime.now()
    total_time = end_time - start_time
    utils.log("Started {0} on-demand instances in {1} seconds" \
      .format(count, total_time.seconds))
    return instance_ids, public_ips, private_ips
Пример #9
0
    def run_instances(self, count, parameters, security_configured):
        """
    Spawns the specified number of EC2 instances using the parameters
    provided. This method is blocking in that it waits until the
    requested VMs are properly booted up. However if the requested
    VMs cannot be procured within 1800 seconds, this method will treat
    it as an error and return. (Also see documentation for the BaseAgent
    class)

    Args:
      count               No. of VMs to spawned
      parameters          A dictionary of parameters. This must contain 'keyname',
                          'group', 'image_id' and 'instance_type' parameters.
      security_configured Uses this boolean value as an heuristic to
                          detect brand new AppScale deployments.

    Returns:
      A tuple of the form (instances, public_ips, private_ips)
    """
        image_id = parameters[self.PARAM_IMAGE_ID]
        instance_type = parameters[self.PARAM_INSTANCE_TYPE]
        keyname = parameters[self.PARAM_KEYNAME]
        group = parameters[self.PARAM_GROUP]
        spot = parameters[self.PARAM_SPOT]
        zone = parameters[self.PARAM_ZONE]

        utils.log("Starting {0} machines with machine id {1}, with " \
          "instance type {2}, keyname {3}, in security group {4}, in zone {5}" \
          .format(count, image_id, instance_type, keyname, group, zone))

        start_time = datetime.datetime.now()
        active_public_ips = []
        active_private_ips = []
        active_instances = []

        try:
            attempts = 1
            while True:
                instances = self.__describe_instances(parameters)
                term_instance_info = self.__get_instance_info(
                    instances, 'terminated', keyname)
                if len(term_instance_info[2]):
                    self.handle_failure('One or more nodes started with key {0} have '\
                                        'been terminated'.format(keyname))
                instance_info = self.__get_instance_info(
                    instances, 'running', keyname)
                active_public_ips = instance_info[0]
                active_private_ips = instance_info[1]
                active_instances = instance_info[2]

                # If security has been configured on this agent just now,
                # that's an indication that this is a fresh cloud deployment.
                # As such it's not expected to have any running VMs.
                if len(active_instances) > 0 or security_configured:
                    break
                elif attempts == self.DESCRIBE_INSTANCES_RETRY_COUNT:
                    self.handle_failure('Failed to invoke describe_instances')
                attempts += 1

            conn = self.open_connection(parameters)
            if spot == 'True':
                price = parameters[self.PARAM_SPOT_PRICE]
                conn.request_spot_instances(str(price),
                                            image_id,
                                            key_name=keyname,
                                            security_groups=[group],
                                            instance_type=instance_type,
                                            count=count,
                                            placement=zone)
            else:
                retries_left = self.RUN_INSTANCES_RETRY_COUNT
                while True:
                    try:
                        conn.run_instances(image_id,
                                           count,
                                           count,
                                           key_name=keyname,
                                           security_groups=[group],
                                           instance_type=instance_type,
                                           placement=zone)
                        break
                    except EC2ResponseError as exception:
                        utils.log("Couldn't start {0} instances because of error: {1}. " \
                          "{2} retries left.".format(count, exception.error_message,
                          retries_left))
                        retries_left = -1
                        if retries_left <= 0:
                            self.handle_failure(exception.error_message)
                        utils.sleep(10)

            instance_ids = []
            public_ips = []
            private_ips = []
            utils.sleep(10)
            end_time = datetime.datetime.now() + datetime.timedelta(
                0, self.MAX_VM_CREATION_TIME)
            now = datetime.datetime.now()

            while now < end_time:
                time_left = (end_time - now).seconds
                utils.log('[{0}] {1} seconds left...'.format(now, time_left))
                instances = self.__describe_instances(parameters)
                term_instance_info = self.__get_instance_info(
                    instances, 'terminated', keyname)
                if len(term_instance_info[2]):
                    self.handle_failure('One or more nodes started with key {0} have '\
                                        'been terminated'.format(keyname))
                instance_info = self.__get_instance_info(
                    instances, 'running', keyname)
                public_ips = instance_info[0]
                private_ips = instance_info[1]
                instance_ids = instance_info[2]
                public_ips = utils.diff(public_ips, active_public_ips)
                private_ips = utils.diff(private_ips, active_private_ips)
                instance_ids = utils.diff(instance_ids, active_instances)
                if count == len(public_ips):
                    break
                time.sleep(self.SLEEP_TIME)
                now = datetime.datetime.now()

            if not public_ips:
                self.handle_failure('No public IPs were able to be procured '
                                    'within the time limit')

            if len(public_ips) != count:
                for index in range(0, len(public_ips)):
                    if public_ips[index] == '0.0.0.0':
                        instance_to_term = instance_ids[index]
                        utils.log('Instance {0} failed to get a public IP address and' \
                                  ' is being terminated'.format(instance_to_term))
                        conn.terminate_instances([instance_to_term])

            end_time = datetime.datetime.now()
            total_time = end_time - start_time
            if spot:
                utils.log('TIMING: It took {0} seconds to spawn {1} spot ' \
                          'instances'.format(total_time.seconds, count))
            else:
                utils.log('TIMING: It took {0} seconds to spawn {1} ' \
                          'regular instances'.format(total_time.seconds, count))
            return instance_ids, public_ips, private_ips
        except EC2ResponseError as exception:
            self.handle_failure('EC2 response error while starting VMs: ' +
                                exception.error_message)
        except Exception as exception:
            if isinstance(exception, AgentRuntimeException):
                raise exception
            else:
                self.handle_failure('Error while starting VMs: ' +
                                    exception.message)
Пример #10
0
    def run_instances(self, count, parameters, security_configured):
        """ Starts 'count' instances in Google Compute Engine, and returns once they
    have been started.

    Callers should create a network and attach a firewall to it before using
    this method, or the newly created instances will not have a network and
    firewall to attach to (and thus this method will fail).

    Args:
      count: An int that specifies how many virtual machines should be started.
      parameters: A dict with keys for each parameter needed to connect to
        Google Compute Engine.
      security_configured: Unused, as we assume that the network and firewall
        has already been set up.
    """
        project_id = parameters[self.PARAM_PROJECT]
        image_id = parameters[self.PARAM_IMAGE_ID]
        instance_type = parameters[self.PARAM_INSTANCE_TYPE]
        keyname = parameters[self.PARAM_KEYNAME]
        group = parameters[self.PARAM_GROUP]

        utils.log("Starting {0} machines with machine id {1}, with " \
          "instance type {2}, keyname {3}, in security group {4}".format(count,
          image_id, instance_type, keyname, group))

        # First, see how many instances are running and what their info is.
        start_time = datetime.datetime.now()
        active_public_ips, active_private_ips, active_instances = \
          self.describe_instances(parameters)

        # Construct URLs
        image_url = '{0}{1}/global/images/{2}'.format(self.GCE_URL, project_id,
                                                      image_id)
        project_url = '{0}{1}'.format(self.GCE_URL, project_id)
        machine_type_url = '{0}/global/machineTypes/{1}'.format(
            project_url, instance_type)
        zone_url = '{0}/zones/{1}'.format(project_url, self.DEFAULT_ZONE)
        network_url = '{0}/global/networks/{1}'.format(project_url, group)

        # Construct the request body
        for index in range(count):
            instances = {
                'name':
                "appscale-{0}-{1}".format(group, uuid.uuid4()),
                'machineType':
                machine_type_url,
                'image':
                image_url,
                'networkInterfaces': [{
                    'accessConfigs': [{
                        'type': 'ONE_TO_ONE_NAT',
                        'name': 'External NAT'
                    }],
                    'network':
                    network_url
                }],
                'serviceAccounts': [{
                    'email': self.DEFAULT_SERVICE_EMAIL,
                    'scopes': [self.GCE_SCOPE]
                }]
            }

            # Create the instance
            gce_service, credentials = self.open_connection(parameters)
            http = httplib2.Http()
            auth_http = credentials.authorize(http)
            request = gce_service.instances().insert(project=project_id,
                                                     body=instances,
                                                     zone=self.DEFAULT_ZONE)
            response = request.execute(auth_http)
            utils.log(str(response))
            self.ensure_operation_succeeds(gce_service, auth_http, response,
                                           parameters[self.PARAM_PROJECT])

        instance_ids = []
        public_ips = []
        private_ips = []
        end_time = datetime.datetime.now() + datetime.timedelta(
            0, self.MAX_VM_CREATION_TIME)
        now = datetime.datetime.now()

        while now < end_time:
            time_left = (end_time - now).seconds
            utils.log("Waiting for your instances to start...")
            instance_info = self.describe_instances(parameters)
            public_ips = instance_info[0]
            private_ips = instance_info[1]
            instance_ids = instance_info[2]
            public_ips = utils.diff(public_ips, active_public_ips)
            private_ips = utils.diff(private_ips, active_private_ips)
            instance_ids = utils.diff(instance_ids, active_instances)
            if count == len(public_ips):
                break
            time.sleep(self.SLEEP_TIME)
            now = datetime.datetime.now()

        if not public_ips:
            self.handle_failure('No public IPs were able to be procured '
                                'within the time limit')

        if len(public_ips) != count:
            for index in range(0, len(public_ips)):
                if public_ips[index] == '0.0.0.0':
                    instance_to_term = instance_ids[index]
                    utils.log('Instance {0} failed to get a public IP address'\
                            'and is being terminated'.format(instance_to_term))
                    self.terminate_instances([instance_to_term])

        end_time = datetime.datetime.now()
        total_time = end_time - start_time
        utils.log("Started {0} on-demand instances in {1} seconds" \
          .format(count, total_time.seconds))
        return instance_ids, public_ips, private_ips
Пример #11
0
  def run_instances(self, count, parameters, security_configured):
    """
    Spawn the specified number of EC2 instances using the parameters
    provided. This method relies on the ec2-run-instances command to
    spawn the actual VMs in the cloud. This method is blocking in that
    it waits until the requested VMs are properly booted up. However
    if the requested VMs cannot be procured within 1800 seconds, this
    method will treat it as an error and return. (Also see documentation
    for the BaseAgent class)

    Args:
      count               No. of VMs to spawned
      parameters          A dictionary of parameters. This must contain 'keyname',
                          'group', 'image_id' and 'instance_type' parameters.
      security_configured Uses this boolean value as an heuristic to
                          detect brand new AppScale deployments.

    Returns:
      A tuple of the form (instances, public_ips, private_ips)
    """
    image_id = parameters[self.PARAM_IMAGE_ID]
    instance_type = parameters[self.PARAM_INSTANCE_TYPE]
    keyname = parameters[self.PARAM_KEYNAME]
    group = parameters[self.PARAM_GROUP]
    spot = False

    utils.log('[{0}] [{1}] [{2}] [{3}] [ec2] [{4}] [{5}]'.format(count,
      image_id, instance_type, keyname, group, spot))

    start_time = datetime.datetime.now()
    active_public_ips = []
    active_private_ips = []
    active_instances = []
    if os.environ.has_key('EC2_URL'):
      utils.log('EC2_URL = [{0}]'.format(os.environ['EC2_URL']))
    else:
      utils.log('Warning: EC2_URL environment not found in the process runtime!')
    while True:
      active_public_ips, active_private_ips, active_instances =\
      self.describe_instances(parameters)
      # If security has been configured on this agent just now,
      # that's an indication that this is a fresh cloud deployment.
      # As such it's not expected to have any running VMs.
      if len(active_instances) > 0 or security_configured:
        break

    args = '-k {0} -n {1} --instance-type {2} --group {3} {4}'.format(keyname,
      count, instance_type, group, image_id)
    if spot:
      price = self.get_optimal_spot_price(instance_type)
      command_to_run = '{0}-request-spot-instances -p {1} {2}'.format(self.prefix, price, args)
    else:
      command_to_run = '{0}-run-instances {1}'.format(self.prefix, args)

    while True:
      run_instances = utils.shell(command_to_run)
      utils.log('Run instances says {0}'.format(run_instances))
      status, command_to_run = self.run_instances_response(command_to_run, run_instances)
      if status:
        break
      utils.log('sleepy time')
      utils.sleep(5)

    instances = []
    public_ips = []
    private_ips = []
    utils.sleep(10)

    end_time = datetime.datetime.now() + datetime.timedelta(0, self.MAX_VM_CREATION_TIME)
    now = datetime.datetime.now()
    while now < end_time:
      describe_instances = utils.shell(self.prefix + '-describe-instances 2>&1')
      utils.log('[{0}] {1} seconds left...'.format(now, (end_time - now).seconds))
      utils.log(describe_instances)
      fqdn_regex = re.compile('\s+({0})\s+({0})\s+running\s+{1}\s'.format(self.FQDN_REGEX, keyname))
      instance_regex = re.compile('INSTANCE\s+(i-\w+)')
      all_ip_addresses = utils.flatten(fqdn_regex.findall(describe_instances))
      instances = utils.flatten(instance_regex.findall(describe_instances))
      public_ips, private_ips = self.get_ip_addresses(all_ip_addresses)
      public_ips = utils.diff(public_ips, active_public_ips)
      private_ips = utils.diff(private_ips, active_private_ips)
      instances = utils.diff(instances, active_instances)
      if count == len(public_ips):
        break
      time.sleep(self.SLEEP_TIME)
      now = datetime.datetime.now()

    if not public_ips:
      sys.exit('No public IPs were able to be procured within the time limit')

    if len(public_ips) != count:
      for index in range(0, len(public_ips)):
        if public_ips[index] == '0.0.0.0':
          instance_to_term = instances[index]
          utils.log('Instance {0} failed to get a public IP address and is being terminated'.\
          format(instance_to_term))
          utils.shell(self.prefix + '-terminate-instances ' + instance_to_term)
      pass

    end_time = datetime.datetime.now()
    total_time = end_time - start_time
    if spot:
      utils.log('TIMING: It took {0} seconds to spawn {1} spot instances'.format(
        total_time.seconds, count))
    else:
      utils.log('TIMING: It took {0} seconds to spawn {1} regular instances'.format(
        total_time.seconds, count))
    return instances, public_ips, private_ips
Пример #12
0
			break
		f2 = Image.open(frames[next_frame])

		if f2.mode != "RGB":
			f2 = f2.convert("RGB") 

		f1_levels = get_resolutions(f1, 3)
		f2_levels = get_resolutions(f2, 3)
		f1_levels.append(f1)
		f2_levels.append(f2)


		width, height = f1.size

		# Temporal difference between frames
		dt = diff(f1, f2)

		# Apply sobel and harris to get the best points to track

		start = time()
		dx, dy = sobel(f1, 3)
		print "Sobel: %.2f segundos" % (time() - start)

		start = time()
		corners = harris(dx, dy, width, height, 3)
		print "Harris: %.2f segundos" % (time() - start)

		# Obtain optical flow using pyramidal implemenatation of lukas kanade feature tracker
		start = time()
		optical_flow = lukas_kanade_pyramidal(corners, f1_levels, f2_levels, dx, dy, dt, 3)
		for point, velocity_vector in optical_flow: