def check_csv_row(csv_reader, nb_cols): line_num = 2 # first line after the header for row in csv_reader: if len(row) != nb_cols: msg = "Wrong number of columns on line {}".format(line_num) raise TuttleError(msg) yield row line_num += 1
def static_check(self, process): inputs = [res for res in process.iter_inputs()] outputs = [res for res in process.iter_outputs()] if len(inputs) != 1 \ or len(outputs) != 1 \ or inputs[0].scheme != 'file' \ or outputs[0].scheme != 'sqlite': raise TuttleError( "CSV2SQLite processor {} don't know how to handle his inputs / outputs" .format(process.id))
def exists(self): if not hostname_resolves(self._host): raise TuttleError("Unknown host : \"{}\"... " "Can't check existence of resource {}.".format(self._host, self.url)) object = self._object() try: res = object.get() return True except (ClientError, BotoCoreError) as e: return False
def exists(self): if not hostname_resolves(self._server): raise TuttleError("Unknown database host : \"{}\"... " "Can't check existence of resource {}.".format( self._server, self.url)) conn_string = "host=\'{}\' dbname='{}' port={} ".format( self._server, self._database, self._port) try: db = psycopg2.connect(conn_string) except psycopg2.OperationalError as e: raise TuttleError( "Can't connect to Postgresql database : \"{}\" to " "check existence of resource {}.".format( conn_string, self.url)) try: result = self.pg_object_type(db, self._schema, self._objectname) is not None db.close() except psycopg2.OperationalError as e: return False return result
def exists(self): try: headers = {"User-Agent": USER_AGENT} req = Request(self.url, headers=headers) response = urlopen(req) some_data = response.read(0) except HTTPError as e: if e.code == 404: return False elif e.code == 401: msg = "Can't access {} because a password is needed. Configure a .tuttlepass file to set " \ "authentication for this resource".format(self.url) raise TuttleError(msg) else: msg = "An error occured while accessing {} : \n{}".format(self.url, str(e)) raise TuttleError(msg) except URLError as e: # return False msg = "An error occured while accessing {} : \n{}".format(self.url, str(e)) raise TuttleError(msg) return True
def exists(self): try: req = Request(self._authenticated_url) response = urlopen(req) some_data = response.read(0) except URLError as e: if e.reason.find("550") > -1: return False msg = "An error occured while accessing {} : \n{}".format( self.url, str(e)) raise TuttleError(msg) return True
def signature(self): # There are so many implementations of ftp it's hard to find a common way to even # retrieve the size of the file. That's why we fallback to a short hash try: req = Request(self._authenticated_url) response = urlopen(req) # a hash from the beginning of the resource chunk_32k = response.read(32768) checksum = sha1() checksum.update(chunk_32k) return "sha1-32K: {}".format(checksum.hexdigest()) except URLError as e: return TuttleError( "Can't compute signature for {}. Error was : {}".format( self.url, str(e)))
def remove(self): raise TuttleError("HTTP resources can't be removed !")