def data(self, date=None):
try:
return self._data
except AttributeError:
params = {
'fn': 'GetSerieSensor',
'interv': 'hora',
'valor_nan': 'nan',
'fecha_inicio': '2003-01-01',
's_cod': self.sensor_code,
'fecha_fin': datetime.now().strftime('%Y-%m-%d')
}
if date is not None:
params['fecha_inicio'] = date.strftime('%Y-%m-%d')
elif isinstance(self.first, datetime):
params['fecha_inicio'] = self.first.strftime('%Y-%m-%d')
tr = Trials
while tr:
r = self.session.get(self.url, params=params)
try:
self._data = data(self, strio(r.text))
except:
tr -= 1
else:
break
return self._data
def get(self, *args, **kwargs):
"""http get implementation using the curl cli executable"""
hdr = CurlHeader()
buf = strio()
self._set_defaults()
cmd = self._mkcurlcmd(*args, **kwargs)
status = Uprocess().call(cmd,
close_fds=True,
stderr=Msg.chlderr,
stdout=Msg.chlderr) # call curl
hdr.setvalue_from_file(self._files["header_file"])
hdr.data["X-ND-CURLSTATUS"] = status
if status:
Msg().err("Error: in download: %s" %
str(FileUtil(self._files["error_file"]).getdata('r')))
FileUtil(self._files["output_file"]).remove()
return (hdr, buf)
status_code = self.get_status_code(hdr.data["X-ND-HTTPSTATUS"])
if "header" in kwargs:
hdr.data["X-ND-HEADERS"] = kwargs["header"]
if status_code == 401: # needs authentication
pass
elif 300 <= status_code <= 308: # redirect
pass
elif "ofile" in kwargs:
if status_code == 206 and "resume" in kwargs:
os.rename(self._files["output_file"], kwargs["ofile"])
elif status_code == 416:
if "resume" in kwargs:
kwargs["resume"] = False
(hdr, buf) = self.get(self._files["url"], **kwargs)
elif status_code != 200:
Msg().err("Error: in download: ",
str(hdr.data["X-ND-HTTPSTATUS"]), ": ", str(status))
FileUtil(self._files["output_file"]).remove()
else: # OK downloaded
os.rename(self._files["output_file"], kwargs["ofile"])
if "ofile" not in kwargs:
try:
buf = strio(open(self._files["output_file"], 'rb').read())
except (IOError, OSError):
Msg().err("Error: reading curl output file to buffer")
FileUtil(self._files["output_file"]).remove()
FileUtil(self._files["error_file"]).remove()
FileUtil(self._files["header_file"]).remove()
return (hdr, buf)
def stations(self):
try:
return self._stations
except AttributeError:
tr = Trials
while tr:
r = self.session.get(self.url, params=Station.params)
with strio(r.text) as io:
self.text = [l for l in csv.reader(io)]
try:
self._stations = [
station(self, *l[:7]) for l in self.text
if l[0][0] != '#'
]
except:
tr -= 1
else:
break
return self._stations
def fields(self):
try:
return self._fields
except AttributeError:
params = Field.params.copy()
params.update([('e_cod', self.code)])
tr = Trials
while tr:
r = self.session.get(self.url, params=params)
with strio(r.text) as io:
self.text = [l for l in csv.reader(io)]
try:
self._fields = [
field(self, *l[:7]) for l in self.text
if l[0][0] != '#'
]
except:
tr -= 1
else:
break
return self._fields
def get(self, *args, **kwargs):
"""http get implementation using the PyCurl"""
hdr = CurlHeader()
buf = strio()
pyc = pycurl.Curl()
self._set_defaults(pyc, hdr)
try:
(output_file, filep) = \
self._mkpycurl(pyc, hdr, buf, *args, **kwargs)
Msg().out("curl url: ", self._url, l=Msg.DBG)
Msg().out("curl arg: ", kwargs, l=Msg.DBG)
pyc.perform() # call pyculr
except (IOError, OSError):
return (None, None)
except pycurl.error as error:
# pylint: disable=unbalanced-tuple-unpacking
errno, errstr = error.args
hdr.data["X-ND-CURLSTATUS"] = errno
if not hdr.data["X-ND-HTTPSTATUS"]:
hdr.data["X-ND-HTTPSTATUS"] = errstr
status_code = self.get_status_code(hdr.data["X-ND-HTTPSTATUS"])
if "header" in kwargs:
hdr.data["X-ND-HEADERS"] = kwargs["header"]
if status_code == 401: # needs authentication
pass
elif 300 <= status_code <= 308: # redirect
pass
elif "ofile" in kwargs:
filep.close()
if status_code == 206 and "resume" in kwargs:
pass
elif status_code == 416 and "resume" in kwargs:
kwargs["resume"] = False
(hdr, buf) = self.get(self._url, **kwargs)
elif status_code != 200:
Msg().err("Error: in download: " +
str(hdr.data["X-ND-HTTPSTATUS"]))
FileUtil(output_file).remove()
return (hdr, buf)
def test_05__mkpycurl(self, mock_jdump):
"""Test05 GetURLpyCurl()._mkpycurl()."""
curl_patch = patch("udocker.utils.curl.CurlHeader")
curlhdr = curl_patch.start()
mock_curlhdr = Mock()
curlhdr.return_value = mock_curlhdr
pyc_patch = patch("udocker.utils.curl.pycurl.Curl")
pycurl = pyc_patch.start()
mock_pycurl = Mock()
pycurl.return_value = mock_pycurl
buff = strio()
argl = ["http://host"]
geturl = GetURLpyCurl()
status = geturl._mkpycurl(pycurl, curlhdr, buff, argl)
self.assertTrue(pycurl.setopt.called)
self.assertEqual(status, ("", None))
kargl = {
"post": "pst1",
"sizeonly": True,
"proxy": "http://proxy",
"ctimeout": 1000,
"header": "Authorization: Bearer",
"v": True,
"nobody": True,
"timeout": 50,
}
mock_jdump.return_value = {"post": "pst1"}
geturl = GetURLpyCurl()
status = geturl._mkpycurl(pycurl, curlhdr, buff, argl, kargl)
self.assertTrue(pycurl.setopt.called)
self.assertTrue(curlhdr.sizeonly)
self.assertEqual(status, ("", None))
curlhdr = curl_patch.stop()
pycurl = pyc_patch.stop()
def test2(self):
ejdb = TestOne._ejdb
self.assertEqual(ejdb.isopen, True)
parrot1 = {
"name": "Grenny",
"type": "African Grey",
"male": True,
"age": 1,
"birthdate": datetime.utcnow(),
"likes": ["green color", "night", "toys"],
"extra1": None
}
parrot2 = {
"name": "Bounty",
"type": "Cockatoo",
"male": False,
"age": 15,
"birthdate": datetime.utcnow(),
"likes": ["sugar cane"],
"extra1": None
}
ejdb.save("parrots", *[parrot1, None, parrot2])
self.assertIsInstance(parrot1["_id"], str)
self.assertIsInstance(parrot2["_id"], str)
p2 = ejdb.load("parrots", parrot2["_id"])
self.assertEqual(p2["_id"], parrot2["_id"])
cur = ejdb.find("parrots")
self.assertEqual(len(cur), 2)
self.assertEqual(len(cur[1:]), 1)
self.assertEqual(len(cur[2:]), 0)
cur = ejdb.find("parrots",
{"name": bson.BSON_Regex(("(grenny|bounty)", "i"))},
hints={"$orderby": [("name", 1)]})
self.assertEqual(len(cur), 2)
self.assertEqual(cur[0]["name"], "Bounty")
self.assertEqual(cur[0]["age"], 15)
cur = ejdb.find("parrots", {}, {"name": "Grenny"}, {"name": "Bounty"},
hints={"$orderby": [("name", 1)]})
self.assertEqual(len(cur), 2)
cur = ejdb.find("parrots", {}, {"name": "Grenny"},
hints={"$orderby": [("name", 1)]})
self.assertEqual(len(cur), 1)
sally = {
"name": "Sally",
"mood": "Angry",
}
molly = {"name": "Molly", "mood": "Very angry", "secret": None}
ejdb.save("birds", *[sally, molly])
logbuf = strio()
ejdb.find("birds", {"name": "Molly"}, log=logbuf)
self.assertTrue(logbuf.getvalue().find("RUN FULLSCAN") != -1)
ejdb.ensureStringIndex("birds", "name")
logbuf = strio()
ejdb.find("birds", {"name": "Molly"}, log=logbuf)
self.assertTrue(logbuf.getvalue().find("MAIN IDX: 'sname'") != -1)
self.assertTrue(logbuf.getvalue().find("RUN FULLSCAN") == -1)
##print("dbmeta=%s" % ejdb.dbmeta())
bar = {"foo": "bar"}
self.assertEqual(ejdb.isactivetx("bars"), False)
ejdb.begintx("bars")
self.assertEqual(ejdb.isactivetx("bars"), True)
ejdb.save("bars", bar)
self.assertTrue(bar["_id"] is not None)
ejdb.abortx("bars")
self.assertTrue(ejdb.load("bars", bar["_id"]) is None)
ejdb.begintx("bars")
ejdb.save("bars", bar)
self.assertTrue(ejdb.load("bars", bar["_id"]) is not None)
self.assertEqual(ejdb.isactivetx("bars"), True)
ejdb.commitx("bars")
self.assertEqual(ejdb.isactivetx("bars"), False)
self.assertTrue(ejdb.load("bars", bar["_id"]) is not None)
ejdb.update("upsertcoll", {"foo": "bar", "$upsert": {"foo": "bar"}})
self.assertTrue(ejdb.findOne("upsertcoll", {"foo": "bar"}) is not None)
def test2(self):
ejdb = TestOne._ejdb
self.assertEqual(ejdb.isopen, True)
parrot1 = {
"name": "Grenny",
"type": "African Grey",
"male": True,
"age": 1,
"birthdate": datetime.utcnow(),
"likes": ["green color", "night", "toys"],
"extra1": None
}
parrot2 = {
"name": "Bounty",
"type": "Cockatoo",
"male": False,
"age": 15,
"birthdate": datetime.utcnow(),
"likes": ["sugar cane"],
"extra1": None
}
ejdb.save("parrots", *[parrot1, None, parrot2])
self.assertEqual(type(parrot1["_id"]).__name__, "str" if PY3 else "unicode")
self.assertEqual(type(parrot2["_id"]).__name__, "str" if PY3 else "unicode")
p2 = ejdb.load("parrots", parrot2["_id"])
self.assertEqual(p2["_id"], parrot2["_id"])
cur = ejdb.find("parrots")
self.assertEqual(len(cur), 2)
self.assertEqual(len(cur[1:]), 1)
self.assertEqual(len(cur[2:]), 0)
cur = ejdb.find("parrots",
{"name": bson.BSON_Regex(("(grenny|bounty)", "i"))},
hints={"$orderby": [("name", 1)]})
self.assertEqual(len(cur), 2)
self.assertEqual(cur[0]["name"], "Bounty")
self.assertEqual(cur[0]["age"], 15)
cur = ejdb.find("parrots", {}, {"name": "Grenny"}, {"name": "Bounty"},
hints={"$orderby": [("name", 1)]})
self.assertEqual(len(cur), 2)
cur = ejdb.find("parrots", {}, {"name": "Grenny"},
hints={"$orderby": [("name", 1)]})
self.assertEqual(len(cur), 1)
sally = {
"name": "Sally",
"mood": "Angry",
}
molly = {
"name": "Molly",
"mood": "Very angry",
"secret": None
}
ejdb.save("birds", *[sally, molly])
logbuf = strio()
ejdb.find("birds", {"name": "Molly"}, log=logbuf)
#print("LB=%s" % logbuf.getvalue())
self.assertTrue(logbuf.getvalue().find("RUN FULLSCAN") != -1)
ejdb.ensureStringIndex("birds", "name")
logbuf = strio()
ejdb.find("birds", {"name": "Molly"}, log=logbuf)
self.assertTrue(logbuf.getvalue().find("MAIN IDX: 'sname'") != -1)
self.assertTrue(logbuf.getvalue().find("RUN FULLSCAN") == -1)
##print("dbmeta=%s" % ejdb.dbmeta())
bar = {
"foo": "bar"
}
self.assertEqual(ejdb.isactivetx("bars"), False)
ejdb.begintx("bars")
self.assertEqual(ejdb.isactivetx("bars"), True)
ejdb.save("bars", bar)
self.assertTrue(bar["_id"] is not None)
ejdb.abortx("bars")
self.assertTrue(ejdb.load("bars", bar["_id"]) is None)
ejdb.begintx("bars")
ejdb.save("bars", bar)
self.assertTrue(ejdb.load("bars", bar["_id"]) is not None)
self.assertEqual(ejdb.isactivetx("bars"), True)
ejdb.commitx("bars")
self.assertEqual(ejdb.isactivetx("bars"), False)
self.assertTrue(ejdb.load("bars", bar["_id"]) is not None)
ejdb.update("upsertcoll",
{"foo": "bar", "$upsert": {"foo": "bar"}})
self.assertTrue(ejdb.findOne("upsertcoll", {"foo": "bar"}) is not None)
cmd = {
"ping": {}
}
cmdret = ejdb.command(cmd)
self.assertIsNotNone(cmdret)
self.assertEqual(cmdret["log"], "pong")
