def check_csv_row(csv_reader, nb_cols):
line_num = 2 # first line after the header
for row in csv_reader:
if len(row) != nb_cols:
msg = "Wrong number of columns on line {}".format(line_num)
raise TuttleError(msg)
yield row
line_num += 1
def static_check(self, process):
inputs = [res for res in process.iter_inputs()]
outputs = [res for res in process.iter_outputs()]
if len(inputs) != 1 \
or len(outputs) != 1 \
or inputs[0].scheme != 'file' \
or outputs[0].scheme != 'sqlite':
raise TuttleError(
"CSV2SQLite processor {} don't know how to handle his inputs / outputs"
.format(process.id))
def exists(self):
if not hostname_resolves(self._host):
raise TuttleError("Unknown host : \"{}\"... "
"Can't check existence of resource {}.".format(self._host, self.url))
object = self._object()
try:
res = object.get()
return True
except (ClientError, BotoCoreError) as e:
return False
def exists(self):
if not hostname_resolves(self._server):
raise TuttleError("Unknown database host : \"{}\"... "
"Can't check existence of resource {}.".format(
self._server, self.url))
conn_string = "host=\'{}\' dbname='{}' port={} ".format(
self._server, self._database, self._port)
try:
db = psycopg2.connect(conn_string)
except psycopg2.OperationalError as e:
raise TuttleError(
"Can't connect to Postgresql database : \"{}\" to "
"check existence of resource {}.".format(
conn_string, self.url))
try:
result = self.pg_object_type(db, self._schema,
self._objectname) is not None
db.close()
except psycopg2.OperationalError as e:
return False
return result
def exists(self):
try:
headers = {"User-Agent": USER_AGENT}
req = Request(self.url, headers=headers)
response = urlopen(req)
some_data = response.read(0)
except HTTPError as e:
if e.code == 404:
return False
elif e.code == 401:
msg = "Can't access {} because a password is needed. Configure a .tuttlepass file to set " \
"authentication for this resource".format(self.url)
raise TuttleError(msg)
else:
msg = "An error occured while accessing {} : \n{}".format(self.url, str(e))
raise TuttleError(msg)
except URLError as e:
# return False
msg = "An error occured while accessing {} : \n{}".format(self.url, str(e))
raise TuttleError(msg)
return True
def exists(self):
try:
req = Request(self._authenticated_url)
response = urlopen(req)
some_data = response.read(0)
except URLError as e:
if e.reason.find("550") > -1:
return False
msg = "An error occured while accessing {} : \n{}".format(
self.url, str(e))
raise TuttleError(msg)
return True
def signature(self):
# There are so many implementations of ftp it's hard to find a common way to even
# retrieve the size of the file. That's why we fallback to a short hash
try:
req = Request(self._authenticated_url)
response = urlopen(req)
# a hash from the beginning of the resource
chunk_32k = response.read(32768)
checksum = sha1()
checksum.update(chunk_32k)
return "sha1-32K: {}".format(checksum.hexdigest())
except URLError as e:
return TuttleError(
"Can't compute signature for {}. Error was : {}".format(
self.url, str(e)))
def remove(self):
raise TuttleError("HTTP resources can't be removed !")