def download(self): """ Just upload one file using Boto3 :param bucket: :param key: :param filepath: :return: """ log = Logger('S3FileDownload') # Make a directory if that's needed dirpath = os.path.dirname(self.abspath) if not os.path.exists(dirpath): try: os.makedirs(dirpath) except Exception as e: raise Exception( "ERROR: Directory `{0}` could not be created.".format( dirpath)) log.info("Downloading: {0} ==> ".format(self.fullkey)) # This step prints straight to stdout and does not log self.s3.download(self.fullkey, self.abspath, size=self.s3size) print "" log.debug("Download Completed: {0}".format(self.abspath))
def upload(self): """ Just upload one file using Boto3 :param bucket: :param key: :param filepath: :return: """ log = Logger('S3FileUpload') log.info("Uploading: {0} ==> s3://{1}/{2}".format( self.abspath, self.bucket, self.fullkey)) # This step prints straight to stdout and does not log self.s3.upload(self.abspath, self.fullkey) print "" log.debug("Upload Completed: {0}".format(self.abspath))
def localProductWalker(projroot, filedict, currentdir=""): """ This method has a similar recursive structure to s3FolderUpload but we're keeping it separate since it is only used to visualize the files in this folder :param rootDir: :param first: :return: """ log = Logger('localProdWalk') for pathseg in os.listdir(os.path.join(projroot, currentdir)): spaces = len(currentdir) * ' ' + '/' # Remember to sanitize for slash unity. We write unix separators # and then translate back to widnows when we need it. relpath = os.path.join(currentdir, pathseg).replace('\\', '/') abspath = os.path.join(projroot, relpath).replace('\\', '/') if os.path.isfile(abspath): log.debug(spaces + relpath) filedict[relpath] = {'src': abspath} elif os.path.isdir(abspath): log.debug(spaces + pathseg + '/') localProductWalker(projroot, filedict, relpath)
class S3Operation: """ A Simple class for storing src/dst file information and the operation we need to perform """ class FileOps: # Kind of an enumeration DELETE_REMOTE = "Delete Remote" DELETE_LOCAL = "Delete Local" UPLOAD = "Upload" DOWNLOAD = "Download" IGNORE = "Ignore" class Direction: # Kind of an enumeration UP = "up" DOWN = "down" class FileState: # Kind of an enumeration LOCALONLY = "Local-Only" REMOTEONLY = "Remote-Only" UPDATENEEDED = "Update Needed" SAME = "Files Match" def __init__(self, key, fileobj, conf): """ :param key: The relative key/path of the file in question :param fileobj: the file object with 'src' and 'dst' :param conf: the configuration dictionary """ self.log = Logger('S3Ops') self.s3 = Transfer(conf['bucket']) self.key = key # Set some sensible defaults self.filestate = self.FileState.SAME self.op = self.FileOps.IGNORE self.delete = conf['delete'] self.force = conf['force'] self.localroot = conf['localroot'] self.bucket = conf['bucket'] self.direction = conf['direction'] self.keyprefix = conf['keyprefix'] self.s3size = 0 # And the final paths we use: self.abspath = self.getAbsLocalPath() self.fullkey = self.getS3Key() # The remote size (if it exists) helps us figure out percent done if 'dst' in fileobj: self.s3size = fileobj['dst']['Size'] # Figure out what we have if 'src' in fileobj and 'dst' not in fileobj: self.filestate = self.FileState.LOCALONLY if 'src' not in fileobj and 'dst' in fileobj: self.filestate = self.FileState.REMOTEONLY if 'src' in fileobj and 'dst' in fileobj: if s3issame(fileobj['src'], fileobj['dst']): self.filestate = self.FileState.SAME else: self.filestate = self.FileState.UPDATENEEDED # The Upload Case # ------------------------------ if self.direction == self.Direction.UP: # Two cases for uploading the file: New file or different file if self.filestate == self.FileState.LOCALONLY or self.filestate == self.FileState.UPDATENEEDED: self.op = self.FileOps.UPLOAD # If we've requested a force, do the upload anyway elif self.FileState.SAME and self.force: self.op = self.FileOps.UPLOAD # If the remote is there but the local is not and we're uploading then clean up the remote # this requires thed delete flag be set elif self.filestate == self.FileState.REMOTEONLY and self.delete: self.op = self.FileOps.DELETE_REMOTE # The Download Case # ------------------------------ elif self.direction == self.Direction.DOWN: if self.filestate == self.FileState.REMOTEONLY or self.filestate == self.FileState.UPDATENEEDED: self.op = self.FileOps.DOWNLOAD # If we've requested a force, do the download anyway elif self.FileState.SAME and self.force: self.op = self.FileOps.DOWNLOAD # If the local is there but the remote is not and we're downloading then clean up the local # this requires thed delete flag be set elif self.filestate == self.FileState.LOCALONLY and self.delete: self.op = self.FileOps.DELETE_LOCAL self.log.info(str(self)) def getS3Key(self): # Not using path.join because can't be guaranteed a unix system return "{1}/{2}".format(self.bucket, self.keyprefix, self.key) def getAbsLocalPath(self): # Not using path.join because can't be guaranteed a unix system return os.path.join(self.localroot, self.key) def execute(self): """ Actually run the command to upload/download/delete the file :return: """ if self.op == self.FileOps.IGNORE: self.log.info(" [{0}] {1}: Nothing to do. Continuing.".format( self.op, self.key)) elif self.op == self.FileOps.UPLOAD: self.upload() elif self.op == self.FileOps.DOWNLOAD: self.download() elif self.op == self.FileOps.DELETE_LOCAL: self.delete_local() elif self.op == self.FileOps.DELETE_REMOTE: self.delete_remote() def __repr__(self): """ When we print this class as a string this is what we output """ forcestr = "(FORCE)" if self.force else "" opstr = "{0:12s} ={2}=> {1:10s}".format(self.filestate, self.op, forcestr) return "./{1:60s} [ {0:21s} ]".format(opstr.strip(), self.key) def delete_remote(self): """ Delete a Remote file """ self.log.info("Deleting: {0} ==> ".format(self.fullkey)) # This step prints straight to stdout and does not log self.s3.delete(self.fullkey) self.log.debug("S3 Deletion Completed: {0}".format(self.fullkey)) def delete_local(self): """ Delete a local file """ dirname = os.path.dirname(self.abspath) os.remove(self.abspath) self.log.info("Deleting Local file: {0} ==> ".format(self.abspath)) # now walk backwards and clean up empty folders try: os.removedirs(dirname) self.log.debug('Cleaning up folders: {0}'.format(dirname)) except: self.log.debug( 'Folder cleanup stopped since there were still files: {0}'. format(dirname)) pass self.log.debug("Local Deletion Completed: {0}".format(self.abspath)) def download(self): """ Just upload one file using Boto3 :param bucket: :param key: :param filepath: :return: """ log = Logger('S3FileDownload') # Make a directory if that's needed dirpath = os.path.dirname(self.abspath) if not os.path.exists(dirpath): try: os.makedirs(dirpath) except Exception as e: raise Exception( "ERROR: Directory `{0}` could not be created.".format( dirpath)) log.info("Downloading: {0} ==> ".format(self.fullkey)) # This step prints straight to stdout and does not log self.s3.download(self.fullkey, self.abspath, size=self.s3size) print "" log.debug("Download Completed: {0}".format(self.abspath)) def upload(self): """ Just upload one file using Boto3 :param bucket: :param key: :param filepath: :return: """ log = Logger('S3FileUpload') log.info("Uploading: {0} ==> s3://{1}/{2}".format( self.abspath, self.bucket, self.fullkey)) # This step prints straight to stdout and does not log self.s3.upload(self.abspath, self.fullkey) print "" log.debug("Upload Completed: {0}".format(self.abspath))