def data(self, date=None): try: return self._data except AttributeError: params = { 'fn': 'GetSerieSensor', 'interv': 'hora', 'valor_nan': 'nan', 'fecha_inicio': '2003-01-01', 's_cod': self.sensor_code, 'fecha_fin': datetime.now().strftime('%Y-%m-%d') } if date is not None: params['fecha_inicio'] = date.strftime('%Y-%m-%d') elif isinstance(self.first, datetime): params['fecha_inicio'] = self.first.strftime('%Y-%m-%d') tr = Trials while tr: r = self.session.get(self.url, params=params) try: self._data = data(self, strio(r.text)) except: tr -= 1 else: break return self._data
def get(self, *args, **kwargs): """http get implementation using the curl cli executable""" hdr = CurlHeader() buf = strio() self._set_defaults() cmd = self._mkcurlcmd(*args, **kwargs) status = Uprocess().call(cmd, close_fds=True, stderr=Msg.chlderr, stdout=Msg.chlderr) # call curl hdr.setvalue_from_file(self._files["header_file"]) hdr.data["X-ND-CURLSTATUS"] = status if status: Msg().err("Error: in download: %s" % str(FileUtil(self._files["error_file"]).getdata('r'))) FileUtil(self._files["output_file"]).remove() return (hdr, buf) status_code = self.get_status_code(hdr.data["X-ND-HTTPSTATUS"]) if "header" in kwargs: hdr.data["X-ND-HEADERS"] = kwargs["header"] if status_code == 401: # needs authentication pass elif 300 <= status_code <= 308: # redirect pass elif "ofile" in kwargs: if status_code == 206 and "resume" in kwargs: os.rename(self._files["output_file"], kwargs["ofile"]) elif status_code == 416: if "resume" in kwargs: kwargs["resume"] = False (hdr, buf) = self.get(self._files["url"], **kwargs) elif status_code != 200: Msg().err("Error: in download: ", str(hdr.data["X-ND-HTTPSTATUS"]), ": ", str(status)) FileUtil(self._files["output_file"]).remove() else: # OK downloaded os.rename(self._files["output_file"], kwargs["ofile"]) if "ofile" not in kwargs: try: buf = strio(open(self._files["output_file"], 'rb').read()) except (IOError, OSError): Msg().err("Error: reading curl output file to buffer") FileUtil(self._files["output_file"]).remove() FileUtil(self._files["error_file"]).remove() FileUtil(self._files["header_file"]).remove() return (hdr, buf)
def stations(self): try: return self._stations except AttributeError: tr = Trials while tr: r = self.session.get(self.url, params=Station.params) with strio(r.text) as io: self.text = [l for l in csv.reader(io)] try: self._stations = [ station(self, *l[:7]) for l in self.text if l[0][0] != '#' ] except: tr -= 1 else: break return self._stations
def fields(self): try: return self._fields except AttributeError: params = Field.params.copy() params.update([('e_cod', self.code)]) tr = Trials while tr: r = self.session.get(self.url, params=params) with strio(r.text) as io: self.text = [l for l in csv.reader(io)] try: self._fields = [ field(self, *l[:7]) for l in self.text if l[0][0] != '#' ] except: tr -= 1 else: break return self._fields
def get(self, *args, **kwargs): """http get implementation using the PyCurl""" hdr = CurlHeader() buf = strio() pyc = pycurl.Curl() self._set_defaults(pyc, hdr) try: (output_file, filep) = \ self._mkpycurl(pyc, hdr, buf, *args, **kwargs) Msg().out("curl url: ", self._url, l=Msg.DBG) Msg().out("curl arg: ", kwargs, l=Msg.DBG) pyc.perform() # call pyculr except (IOError, OSError): return (None, None) except pycurl.error as error: # pylint: disable=unbalanced-tuple-unpacking errno, errstr = error.args hdr.data["X-ND-CURLSTATUS"] = errno if not hdr.data["X-ND-HTTPSTATUS"]: hdr.data["X-ND-HTTPSTATUS"] = errstr status_code = self.get_status_code(hdr.data["X-ND-HTTPSTATUS"]) if "header" in kwargs: hdr.data["X-ND-HEADERS"] = kwargs["header"] if status_code == 401: # needs authentication pass elif 300 <= status_code <= 308: # redirect pass elif "ofile" in kwargs: filep.close() if status_code == 206 and "resume" in kwargs: pass elif status_code == 416 and "resume" in kwargs: kwargs["resume"] = False (hdr, buf) = self.get(self._url, **kwargs) elif status_code != 200: Msg().err("Error: in download: " + str(hdr.data["X-ND-HTTPSTATUS"])) FileUtil(output_file).remove() return (hdr, buf)
def test_05__mkpycurl(self, mock_jdump): """Test05 GetURLpyCurl()._mkpycurl().""" curl_patch = patch("udocker.utils.curl.CurlHeader") curlhdr = curl_patch.start() mock_curlhdr = Mock() curlhdr.return_value = mock_curlhdr pyc_patch = patch("udocker.utils.curl.pycurl.Curl") pycurl = pyc_patch.start() mock_pycurl = Mock() pycurl.return_value = mock_pycurl buff = strio() argl = ["http://host"] geturl = GetURLpyCurl() status = geturl._mkpycurl(pycurl, curlhdr, buff, argl) self.assertTrue(pycurl.setopt.called) self.assertEqual(status, ("", None)) kargl = { "post": "pst1", "sizeonly": True, "proxy": "http://proxy", "ctimeout": 1000, "header": "Authorization: Bearer", "v": True, "nobody": True, "timeout": 50, } mock_jdump.return_value = {"post": "pst1"} geturl = GetURLpyCurl() status = geturl._mkpycurl(pycurl, curlhdr, buff, argl, kargl) self.assertTrue(pycurl.setopt.called) self.assertTrue(curlhdr.sizeonly) self.assertEqual(status, ("", None)) curlhdr = curl_patch.stop() pycurl = pyc_patch.stop()
def test2(self): ejdb = TestOne._ejdb self.assertEqual(ejdb.isopen, True) parrot1 = { "name": "Grenny", "type": "African Grey", "male": True, "age": 1, "birthdate": datetime.utcnow(), "likes": ["green color", "night", "toys"], "extra1": None } parrot2 = { "name": "Bounty", "type": "Cockatoo", "male": False, "age": 15, "birthdate": datetime.utcnow(), "likes": ["sugar cane"], "extra1": None } ejdb.save("parrots", *[parrot1, None, parrot2]) self.assertIsInstance(parrot1["_id"], str) self.assertIsInstance(parrot2["_id"], str) p2 = ejdb.load("parrots", parrot2["_id"]) self.assertEqual(p2["_id"], parrot2["_id"]) cur = ejdb.find("parrots") self.assertEqual(len(cur), 2) self.assertEqual(len(cur[1:]), 1) self.assertEqual(len(cur[2:]), 0) cur = ejdb.find("parrots", {"name": bson.BSON_Regex(("(grenny|bounty)", "i"))}, hints={"$orderby": [("name", 1)]}) self.assertEqual(len(cur), 2) self.assertEqual(cur[0]["name"], "Bounty") self.assertEqual(cur[0]["age"], 15) cur = ejdb.find("parrots", {}, {"name": "Grenny"}, {"name": "Bounty"}, hints={"$orderby": [("name", 1)]}) self.assertEqual(len(cur), 2) cur = ejdb.find("parrots", {}, {"name": "Grenny"}, hints={"$orderby": [("name", 1)]}) self.assertEqual(len(cur), 1) sally = { "name": "Sally", "mood": "Angry", } molly = {"name": "Molly", "mood": "Very angry", "secret": None} ejdb.save("birds", *[sally, molly]) logbuf = strio() ejdb.find("birds", {"name": "Molly"}, log=logbuf) self.assertTrue(logbuf.getvalue().find("RUN FULLSCAN") != -1) ejdb.ensureStringIndex("birds", "name") logbuf = strio() ejdb.find("birds", {"name": "Molly"}, log=logbuf) self.assertTrue(logbuf.getvalue().find("MAIN IDX: 'sname'") != -1) self.assertTrue(logbuf.getvalue().find("RUN FULLSCAN") == -1) ##print("dbmeta=%s" % ejdb.dbmeta()) bar = {"foo": "bar"} self.assertEqual(ejdb.isactivetx("bars"), False) ejdb.begintx("bars") self.assertEqual(ejdb.isactivetx("bars"), True) ejdb.save("bars", bar) self.assertTrue(bar["_id"] is not None) ejdb.abortx("bars") self.assertTrue(ejdb.load("bars", bar["_id"]) is None) ejdb.begintx("bars") ejdb.save("bars", bar) self.assertTrue(ejdb.load("bars", bar["_id"]) is not None) self.assertEqual(ejdb.isactivetx("bars"), True) ejdb.commitx("bars") self.assertEqual(ejdb.isactivetx("bars"), False) self.assertTrue(ejdb.load("bars", bar["_id"]) is not None) ejdb.update("upsertcoll", {"foo": "bar", "$upsert": {"foo": "bar"}}) self.assertTrue(ejdb.findOne("upsertcoll", {"foo": "bar"}) is not None)
def test2(self): ejdb = TestOne._ejdb self.assertEqual(ejdb.isopen, True) parrot1 = { "name": "Grenny", "type": "African Grey", "male": True, "age": 1, "birthdate": datetime.utcnow(), "likes": ["green color", "night", "toys"], "extra1": None } parrot2 = { "name": "Bounty", "type": "Cockatoo", "male": False, "age": 15, "birthdate": datetime.utcnow(), "likes": ["sugar cane"], "extra1": None } ejdb.save("parrots", *[parrot1, None, parrot2]) self.assertEqual(type(parrot1["_id"]).__name__, "str" if PY3 else "unicode") self.assertEqual(type(parrot2["_id"]).__name__, "str" if PY3 else "unicode") p2 = ejdb.load("parrots", parrot2["_id"]) self.assertEqual(p2["_id"], parrot2["_id"]) cur = ejdb.find("parrots") self.assertEqual(len(cur), 2) self.assertEqual(len(cur[1:]), 1) self.assertEqual(len(cur[2:]), 0) cur = ejdb.find("parrots", {"name": bson.BSON_Regex(("(grenny|bounty)", "i"))}, hints={"$orderby": [("name", 1)]}) self.assertEqual(len(cur), 2) self.assertEqual(cur[0]["name"], "Bounty") self.assertEqual(cur[0]["age"], 15) cur = ejdb.find("parrots", {}, {"name": "Grenny"}, {"name": "Bounty"}, hints={"$orderby": [("name", 1)]}) self.assertEqual(len(cur), 2) cur = ejdb.find("parrots", {}, {"name": "Grenny"}, hints={"$orderby": [("name", 1)]}) self.assertEqual(len(cur), 1) sally = { "name": "Sally", "mood": "Angry", } molly = { "name": "Molly", "mood": "Very angry", "secret": None } ejdb.save("birds", *[sally, molly]) logbuf = strio() ejdb.find("birds", {"name": "Molly"}, log=logbuf) #print("LB=%s" % logbuf.getvalue()) self.assertTrue(logbuf.getvalue().find("RUN FULLSCAN") != -1) ejdb.ensureStringIndex("birds", "name") logbuf = strio() ejdb.find("birds", {"name": "Molly"}, log=logbuf) self.assertTrue(logbuf.getvalue().find("MAIN IDX: 'sname'") != -1) self.assertTrue(logbuf.getvalue().find("RUN FULLSCAN") == -1) ##print("dbmeta=%s" % ejdb.dbmeta()) bar = { "foo": "bar" } self.assertEqual(ejdb.isactivetx("bars"), False) ejdb.begintx("bars") self.assertEqual(ejdb.isactivetx("bars"), True) ejdb.save("bars", bar) self.assertTrue(bar["_id"] is not None) ejdb.abortx("bars") self.assertTrue(ejdb.load("bars", bar["_id"]) is None) ejdb.begintx("bars") ejdb.save("bars", bar) self.assertTrue(ejdb.load("bars", bar["_id"]) is not None) self.assertEqual(ejdb.isactivetx("bars"), True) ejdb.commitx("bars") self.assertEqual(ejdb.isactivetx("bars"), False) self.assertTrue(ejdb.load("bars", bar["_id"]) is not None) ejdb.update("upsertcoll", {"foo": "bar", "$upsert": {"foo": "bar"}}) self.assertTrue(ejdb.findOne("upsertcoll", {"foo": "bar"}) is not None) cmd = { "ping": {} } cmdret = ejdb.command(cmd) self.assertIsNotNone(cmdret) self.assertEqual(cmdret["log"], "pong")