Пример #1
0
def downloadData(sensor_ids,
                 step,
                 download_startdate,
                 download_enddate,
                 outfn=None,
                 pool_len=48):
    """
        Funcion que arme para bajar datos de la api de teracode
        Ej:
        sensor_ids = [...] # Sacar de waypoints.py
        download_startdate = "2015-07-01T00:00:00-00:00"
        download_enddate = "2015-07-12T00:00:01-00:00"
        step = datetime.timedelta(days=2)
        newdata = downloadData (
            sensor_ids, step, download_startdate, download_enddate, outfn="raw_api_01_11.json")
    """
    # vsensids = virtsens["id_sensor"].unique()
    token = config.api['token']
    host = config.api['host']
    urltpl = "https://{0}/api/data/%s?token=%s&fecha_desde=%s&fecha_hasta=%s".format(
        host)

    # end = dateutil.parser.parse(download_enddate)
    start = download_startdate
    end = download_enddate
    urls = []
    if step > (download_enddate - download_startdate):
        step = download_enddate - download_startdate
    else:
        logger.info(
            "condicion step sin cumplir step:{0} enddate:{1} startdate:{2}".
            format(step, download_enddate, download_startdate))
    while start < end:
        startdate, enddate = start, start + step
        for sensor_id in sensor_ids:
            # startdate, enddate, sensor_id
            url = urltpl % (sensor_id, token,
                            startdate.strftime("%Y-%m-%dT%H:%M:%S-03:00"),
                            enddate.strftime("%Y-%m-%dT%H:%M:%S-03:00"))
            if not url in urls:
                urls.append(url)
        start += step
    """cambiar funcion map por api_sensores_fake"""
    #alldata = map(api_sensores_fake, urls)
    try:
        pool = multiprocessing.Pool(pool_len)
        alldata = pool.map(getData, urls)
    except Exception, e:
        logger.error("pool multiprocessing, error:", traceback=True)
        send_email_error("pool multiprocessing, error: {0}".format(e))
        alldata = []
Пример #2
0
 def createDBEngine(self):
     """
             http://docs.sqlalchemy.org/en/latest/core/engines.html?highlight=create_engine#sqlalchemy.create_engine
             engine = sqlalchemy.create_engine(
                 "postgres://postgres@/postgres")
             engine = sqlalchemy.create_engine("sqlite:///analysis.db")
     """
     try:
         self.__engine = create_engine(
             self.cfg.db_url, echo=self.debug, pool_timeout=60, echo_pool=self.debug_pool)
         self.Base.prepare(self.__engine, reflect=True)
         self.__c = True
     except Exception, e:
         msg_error = "OperationalError: not connect {0} , traceback:{1}".format(self.cfg.db_url, e)
         logger.error("OperationalError: not connect {0} ".format(self.cfg.db_url), traceback=True)
         send_email_error(msg_error)
Пример #3
0
def downloadData(sensor_ids, step, download_startdate, download_enddate, outfn=None, pool_len=48):
    """
        Funcion que arme para bajar datos de la api de teracode
        Ej:
        sensor_ids = [...] # Sacar de waypoints.py
        download_startdate = "2015-07-01T00:00:00-00:00"
        download_enddate = "2015-07-12T00:00:01-00:00"
        step = datetime.timedelta(days=2)
        newdata = downloadData (
            sensor_ids, step, download_startdate, download_enddate, outfn="raw_api_01_11.json")
    """
    # vsensids = virtsens["id_sensor"].unique()
    token = config.api['token']
    host = config.api['host']
    urltpl = "{0}/api/data/%s?token=%s&fecha_desde=%s&fecha_hasta=%s".format(
        host)

    # end = dateutil.parser.parse(download_enddate)
    start = download_startdate
    end = download_enddate
    urls = []
    if step > (download_enddate - download_startdate):
        step = download_enddate - download_startdate
    else:
        logger.info("condicion step sin cumplir step:{0} enddate:{1} startdate:{2}".format(
            step, download_enddate, download_startdate))
    while start < end:
        startdate, enddate = start, start + step
        for sensor_id in sensor_ids:
            # startdate, enddate, sensor_id
            url = urltpl % (sensor_id, token, startdate.strftime(
                "%Y-%m-%dT%H:%M:%S-03:00"), enddate.strftime("%Y-%m-%dT%H:%M:%S-03:00"))
            if not url in urls:
                urls.append(url)
        start += step

    """cambiar funcion map por api_sensores_fake"""
    #alldata = map(api_sensores_fake, urls)
    try:
        pool = multiprocessing.Pool(pool_len)
        alldata = pool.map(getData, urls)
    except Exception, e:
        logger.error("pool multiprocessing, error:", traceback=True)
        send_email_error("pool multiprocessing, error: {0}".format(e))
        alldata = []
Пример #4
0
 def createDBEngine(self):
     """
             http://docs.sqlalchemy.org/en/latest/core/engines.html?highlight=create_engine#sqlalchemy.create_engine
             engine = sqlalchemy.create_engine(
                 "postgres://postgres@/postgres")
             engine = sqlalchemy.create_engine("sqlite:///analysis.db")
     """
     try:
         self.__engine = create_engine(self.cfg.db_url,
                                       echo=self.debug,
                                       pool_timeout=60,
                                       echo_pool=self.debug_pool)
         self.Base.prepare(self.__engine, reflect=True)
         self.__c = True
     except Exception, e:
         msg_error = "OperationalError: not connect {0} , traceback:{1}".format(
             self.cfg.db_url, e)
         logger.error("OperationalError: not connect {0} ".format(
             self.cfg.db_url),
                      traceback=True)
         send_email_error(msg_error)
Пример #5
0
        try:
            response = requests.get(url)
            if (response.status_code == 200):
                return response.json()
            else:
                error_msg = "hubo timeout del count:{0} request en {1} codigo:{2}".format(
                    i, url, response.status_code)
                logger.error(error_msg)
        except requests.exceptions.Timeout, e:
            error_msg = "hubo timeout del count:{0} request en {1}".format(
                i, url)
            logger.error(error_msg, traceback=True)
        except:
            return None
        if i == 3:
            send_email_error(error_msg)
            time.sleep(3)
    return None


def downloadData(sensor_ids,
                 step,
                 download_startdate,
                 download_enddate,
                 outfn=None,
                 pool_len=48):
    """
        Funcion que arme para bajar datos de la api de teracode
        Ej:
        sensor_ids = [...] # Sacar de waypoints.py
        download_startdate = "2015-07-01T00:00:00-00:00"
Пример #6
0
        try:
            response = requests.get(url)
            if (response.status_code == 200):
                return response.json()
            else:
                error_msg = "hubo error del count:{0} request en {1} codigo:{2}".format(
                    i, url, response.status_code)
                logger.error(error_msg)
        except requests.exceptions.Timeout, e:
            error_msg = "hubo timeout del count:{0} request en {1}".format(
                i, url)
            logger.error(error_msg, traceback=True)
        except:
            return None
        if i == 3:
            send_email_error(error_msg)
            time.sleep(3)
    return None


def downloadData(sensor_ids, step, download_startdate, download_enddate, outfn=None, pool_len=48):
    """
        Funcion que arme para bajar datos de la api de teracode
        Ej:
        sensor_ids = [...] # Sacar de waypoints.py
        download_startdate = "2015-07-01T00:00:00-00:00"
        download_enddate = "2015-07-12T00:00:01-00:00"
        step = datetime.timedelta(days=2)
        newdata = downloadData (
            sensor_ids, step, download_startdate, download_enddate, outfn="raw_api_01_11.json")
    """