def _getData(cls, regionName, instanceID, metricName, stats, unit, period,
                 startTime, endTime):
        """ For experimentation """
        cw = cloudwatch.connect_to_region(region_name=regionName,
                                          **getAWSCredentials())

        data = cw.get_metric_statistics(period=period,
                                        start_time=startTime,
                                        end_time=endTime,
                                        metric_name=metricName,
                                        namespace="AWS/EC2",
                                        statistics=stats,
                                        dimensions=dict(InstanceId=instanceID),
                                        unit=unit)

        return data
  def _getData(cls, regionName, instanceID, metricName, stats, unit, period,
              startTime, endTime):
    """ For experimentation """
    cw = cloudwatch.connect_to_region(region_name=regionName,
                                      **getAWSCredentials())

    data = cw.get_metric_statistics(
      period=period,
      start_time=startTime,
      end_time=endTime,
      metric_name=metricName,
      namespace="AWS/EC2",
      statistics=stats,
      dimensions=dict(InstanceId=instanceID),
      unit=unit)

    return data
def _collectMetrics(task):
    """ Executed via multiprocessing Pool: Collect metric data

  :param task: a _MetricCollectionTask object

  :returns: a _MetricCollectionTaskResult object
  """
    log = getExtendedLogger(_MODULE_NAME + ".COLLECTOR")

    taskResult = _MetricCollectionTaskResult(refID=task.refID,
                                             metricID=task.metricID,
                                             instanceID=task.instanceID)
    try:
        if task.metricName == "InstanceCount":
            timestamp = datetime.utcnow().replace(second=0, microsecond=0)
            taskResult.data = (MetricRecord(timestamp=timestamp, value=1.0), )
        else:
            backoffSec = 0.75
            backoffGrowthFactor = 1.5
            maxBackoffSec = 5
            rawdata = None
            while rawdata is None:
                try:
                    cw = cloudwatch.connect_to_region(region_name=task.region,
                                                      **getAWSCredentials())

                    rawdata = cw.get_metric_statistics(
                        period=task.period,
                        start_time=task.timeRange.start,
                        end_time=task.timeRange.end,
                        metric_name=task.metricName,
                        namespace="AWS/EC2",
                        statistics=task.stats,
                        dimensions=dict(InstanceId=task.instanceID),
                        unit=task.unit)
                except BotoServerError as e:
                    if e.status == 400 and e.error_code == "Throttling":
                        # TODO: unit-test

                        log.info("Throttling: %r", e)

                        if backoffSec <= maxBackoffSec:
                            time.sleep(backoffSec)
                            backoffSec *= backoffGrowthFactor
                        else:
                            raise app_exceptions.MetricThrottleError(repr(e))
                    else:
                        raise

            # Sort the data by timestamp in ascending order
            rawdata.sort(key=lambda row: row["Timestamp"])

            # Convert raw data to MetricRecord objects
            def getValue(rawDataItem, stats):
                if isinstance(stats, basestring):
                    return rawDataItem[stats]
                else:
                    return dict((field, rawDataItem[field]) for field in stats)

            taskResult.data = tuple(
                MetricRecord(timestamp=rawItem["Timestamp"],
                             value=getValue(rawItem, task.stats))
                for rawItem in rawdata)

    except Exception as e:  # pylint: disable=W0703
        log.exception("Error in task=%r", task)
        taskResult.exception = e
    finally:
        taskResult.duration = time.time() - taskResult.creationTime

    return taskResult
def _collectMetrics(task):
  """ Executed via multiprocessing Pool: Collect metric data

  :param task: a _MetricCollectionTask object

  :returns: a _MetricCollectionTaskResult object
  """
  log = getExtendedLogger(_MODULE_NAME + ".COLLECTOR")

  taskResult = _MetricCollectionTaskResult(
    refID=task.refID,
    metricID=task.metricID,
    instanceID=task.instanceID)
  try:
    if task.metricName == "InstanceCount":
      timestamp = datetime.utcnow().replace(second=0, microsecond=0)
      taskResult.data = (MetricRecord(timestamp=timestamp,
                                      value=1.0),)
    else:
      backoffSec = 0.75
      backoffGrowthFactor = 1.5
      maxBackoffSec = 5
      rawdata = None
      while rawdata is None:
        try:
          cw = cloudwatch.connect_to_region(region_name=task.region,
                                            **getAWSCredentials())

          rawdata = cw.get_metric_statistics(
            period=task.period,
            start_time=task.timeRange.start,
            end_time=task.timeRange.end,
            metric_name=task.metricName,
            namespace="AWS/EC2",
            statistics=task.stats,
            dimensions=dict(InstanceId=task.instanceID),
            unit=task.unit)
        except BotoServerError as e:
          if e.status == 400 and e.error_code == "Throttling":
            # TODO: unit-test

            log.info("Throttling: %r", e)

            if backoffSec <= maxBackoffSec:
              time.sleep(backoffSec)
              backoffSec *= backoffGrowthFactor
            else:
              raise app_exceptions.MetricThrottleError(repr(e))
          else:
            raise

      # Sort the data by timestamp in ascending order
      rawdata.sort(key=lambda row: row["Timestamp"])

      # Convert raw data to MetricRecord objects
      def getValue(rawDataItem, stats):
        if isinstance(stats, basestring):
          return rawDataItem[stats]
        else:
          return dict((field, rawDataItem[field]) for field in stats)

      taskResult.data = tuple(
        MetricRecord(timestamp=rawItem["Timestamp"],
                     value=getValue(rawItem, task.stats))
      for rawItem in rawdata)

  except Exception as e:  # pylint: disable=W0703
    log.exception("Error in task=%r", task)
    taskResult.exception = e
  finally:
    taskResult.duration = time.time() - taskResult.creationTime

  return taskResult