class Command(BaseCommand): help = ("List all the items in a container to stdout.\n\n" "We recommend you run it like this:\n" " ./manage.py container_list <container> | pv --line-mode > <container>.list\n\n" "pv is Pipe Viewer: http://www.ivarch.com/programs/pv.shtml") args = "[container_name]" def handle(self, *args, **options): """ Lists all the items in a container to stdout. """ self._connection = Auth()._get_connection() if len(args) == 0: containers = self._connection.list_containers() if not containers: print("No containers were found for this account.") elif len(args) == 1: containers = self._connection.list_container_object_names(args[0]) if not containers: print("No matching container found.") else: raise CommandError("Pass one and only one [container_name] as an argument") for container in containers: print(container)
class Command(BaseCommand): help = "Display info for containers" args = "[container_name container_name ...]" option_list = BaseCommand.option_list + ( optparse.make_option( "-n", "--name", action="store_true", dest="name", default=False), optparse.make_option( "-c", "--count", action="store_true", dest="count", default=False), optparse.make_option( "-s", "--size", action="store_true", dest="size", default=False), optparse.make_option( "-u", "--uri", action="store_true", dest="uri", default=False)) def handle(self, *args, **options): self._connection = Auth()._get_connection() container_names = self._connection.list_container_names() if args: matches = [] for container_name in container_names: if container_name in args: matches.append(container_name) container_names = matches if not container_names: print("No containers found.") return if not args: account_details = self._connection.get_account_details() print("container_count | object_count | bytes_used") print("{0}, {1}, {2}\n".format( account_details["container_count"], account_details["object_count"], account_details["bytes_used"], )) opts = ["name", "count", "size", "uri"] output = [o for o in opts if options.get(o)] if output: print(" | ".join(output)) else: print(" | ".join(opts)) for container_name in container_names: container = self._connection.get_container(container_name) info = { "name": container.name, "count": container.object_count, "size": container.total_bytes, "cdn_enabled": container.cdn_enabled, "uri": container.cdn_uri if container.cdn_enabled else None, } output = [str(info[o]) for o in opts if options.get(o)] if not output: output = [str(info[o]) for o in opts] print(", ".join(output))
class Command(BaseCommand): help = "Display info for containers" args = "[container_name container_name ...]" option_list = BaseCommand.option_list + ( optparse.make_option("-n", "--name", action="store_true", dest="name", default=False), optparse.make_option("-c", "--count", action="store_true", dest="count", default=False), optparse.make_option("-s", "--size", action="store_true", dest="size", default=False), optparse.make_option("-u", "--uri", action="store_true", dest="uri", default=False) ) def handle(self, *args, **options): self._connection = Auth()._get_connection() container_names = self._connection.list_container_names() if args: matches = [] for container_name in container_names: if container_name in args: matches.append(container_name) container_names = matches if not container_names: print("No containers found.") return if not args: account_details = self._connection.get_account_details() print("container_count | object_count | bytes_used") print("{0}, {1}, {2}\n".format( account_details["container_count"], account_details["object_count"], account_details["bytes_used"], )) opts = ["name", "count", "size", "uri"] output = [o for o in opts if options.get(o)] if output: print(" | ".join(output)) else: print(" | ".join(opts)) for container_name in container_names: container = self._connection.get_container(container_name) info = { "name": container.name, "count": container.object_count, "size": container.total_bytes, "cdn_enabled": container.cdn_enabled, "uri": container.cdn_uri if container.cdn_enabled else None, } output = [str(info[o]) for o in opts if options.get(o)] if not output: output = [str(info[o]) for o in opts] print(", ".join(output))
def handle(self, *args, **options): if len(args) != 1: raise CommandError("Pass one and only one [container_name] as an argument") self._connection = Auth()._get_connection() container_name = args[0] print("Creating container: {0}".format(container_name)) container = self._connection.create_container(container_name) if options.get("private"): print("Private container: {0}".format(container_name)) container.make_private() else: print("Public container: {0}".format(container_name)) container.make_public(ttl=CUMULUS["TTL"])
class Command(BaseCommand): help = "Delete a container." args = "[container_name]" def add_arguments(self, parser): parser.add_argument('-y', '--yes', action='store_true', default=False, dest='is_yes', help='Assume Yes to confiramtion question') def handle(self, *args, **options): if len(args) != 1: raise CommandError("Pass one and only one [container_name] as an argument") container_name = args[0] if not options.get("is_yes"): is_ok = raw_input("Permanently delete container {0}? [y|N] ".format( container_name)) if not is_ok == "y": raise CommandError("Aborted") print("Connecting") self._connection = Auth()._get_connection() container = self._connection.get_container(container_name) print("Deleting objects from container {0}".format(container_name)) container.delete_all_objects() container.delete() print("Deletion complete")
class Command(BaseCommand): help = "Delete a container." args = "[container_name]" option_list = BaseCommand.option_list + (optparse.make_option( "-y", "--yes", action="store_true", default=False, dest="is_yes", help="Assume Yes to confirmation question"), ) def handle(self, *args, **options): if len(args) != 1: raise CommandError( "Pass one and only one [container_name] as an argument") container_name = args[0] if not options.get("is_yes"): is_ok = raw_input( "Permanently delete container {0}? [y|N] ".format( container_name)) if not is_ok == "y": raise CommandError("Aborted") print("Connecting") self._connection = Auth()._get_connection() container = self._connection.get_container(container_name) print("Deleting objects from container {0}".format(container_name)) container.delete_all_objects() container.delete() print("Deletion complete")
class Command(BaseCommand): help = "Delete a container." args = "[container_name]" option_list = BaseCommand.option_list + ( optparse.make_option("-y", "--yes", action="store_true", default=False, dest="is_yes", help="Assume Yes to confirmation question"),) def handle(self, *args, **options): if len(args) != 1: raise CommandError("Pass one and only one [container_name] as an argument") container_name = args[0] if not options.get("is_yes"): is_ok = raw_input("Permanently delete container {0}? [y|N] ".format( container_name)) if not is_ok == "y": raise CommandError("Aborted") print("Connecting") self._connection = Auth()._get_connection() container = self._connection.get_container(container_name) print("Deleting objects from container {0}".format(container_name)) container.delete_all_objects() container.delete() print("Deletion complete")
class Command(BaseCommand): help = "Delete a container." args = "[container_name]" def add_arguments(self, parser): parser.add_argument('-y', '--yes', action='store_true', default=False, dest='is_yes', help='Assume Yes to confiramtion question') def handle(self, *args, **options): if len(args) != 1: raise CommandError( "Pass one and only one [container_name] as an argument") container_name = args[0] if not options.get("is_yes"): is_ok = raw_input( "Permanently delete container {0}? [y|N] ".format( container_name)) if not is_ok == "y": raise CommandError("Aborted") print("Connecting") self._connection = Auth()._get_connection() container = self._connection.get_container(container_name) print("Deleting objects from container {0}".format(container_name)) container.delete_all_objects() container.delete() print("Deletion complete")
class Command(BaseCommand): help = "Create a container." args = "[container_name]" def add_arguments(self, parser): parser.add_argument('-p', '--private', action='store_true', default=False, dest='private', help='Assume Yes to confiramtion question') def handle(self, *args, **options): if len(args) != 1: raise CommandError( "Pass one and only one [container_name] as an argument") self._connection = Auth()._get_connection() container_name = args[0] print("Creating container: {0}".format(container_name)) container = self._connection.create_container(container_name) if options.get("private"): print("Private container: {0}".format(container_name)) container.make_private() else: print("Public container: {0}".format(container_name)) container.make_public(ttl=CUMULUS["TTL"])
def handle(self, *args, **options): self._connection = Auth()._get_connection() container_names = self._connection.list_container_names() if args: matches = [] for container_name in container_names: if container_name in args: matches.append(container_name) container_names = matches if not container_names: print("No containers found.") return if not args: account_details = self._connection.get_account_details() print("container_count | object_count | bytes_used") print("{0}, {1}, {2}\n".format( account_details["container_count"], account_details["object_count"], account_details["bytes_used"], )) opts = ["name", "count", "size", "uri"] output = [o for o in opts if options.get(o)] if output: print(" | ".join(output)) else: print(" | ".join(opts)) for container_name in container_names: container = self._connection.get_container(container_name) info = { "name": container.name, "count": container.object_count, "size": container.total_bytes, "cdn_enabled": container.cdn_enabled, "uri": container.cdn_uri if container.cdn_enabled else None, } output = [str(info[o]) for o in opts if options.get(o)] if not output: output = [str(info[o]) for o in opts] print(", ".join(output))
def handle_noargs(self, *args, **options): # setup self.set_options(options) self._connection = Auth()._get_connection() self.container = self._connection.get_container(self.container_name) # wipe first if self.wipe: self.wipe_container() # match local files abspaths = self.match_local(self.file_root, self.includes, self.excludes) relpaths = [] for path in abspaths: filename = path.split(self.file_root)[1] if filename.startswith("/"): filename = filename[1:] relpaths.append(filename) if not relpaths: settings_root_prefix = "MEDIA" if self.syncmedia else "STATIC" raise CommandError( "The {0}_ROOT directory is empty " "or all files have been ignored.".format(settings_root_prefix)) for path in abspaths: if not os.path.isfile(path): raise CommandError("Unsupported filetype: {0}.".format(path)) # match cloud objects cloud_objs = self.match_cloud(self.includes, self.excludes) remote_objects = { obj.name: datetime.datetime.strptime(obj.last_modified, "%Y-%m-%dT%H:%M:%S.%f") for obj in self.container.get_objects() } # sync self.upload_files(abspaths, relpaths, remote_objects) self.delete_extra_files(relpaths, cloud_objs) if not self.quiet or self.verbosity > 1: self.print_tally()
def set_options(self, **options): """ Set instance variables based on an options dict """ self.interactive = options['interactive'] self.verbosity = options['verbosity'] self.symlink = options['link'] self.clear = options['clear'] self.dry_run = options['dry_run'] ignore_patterns = options['ignore_patterns'] if options['use_default_ignore_patterns']: ignore_patterns += ['CVS', '.*', '*~'] self.ignore_patterns = list(set(ignore_patterns)) self.post_process = options['post_process'] self.container_name = CUMULUS["CONTAINER"] self._connection = Auth()._get_connection() self.container = self._connection.get_container(self.container_name)
def handle(self, *args, **options): """ Lists all the items in a container to stdout. """ self._connection = Auth()._get_connection() if len(args) == 0: containers = self._connection.list_containers() if not containers: print("No containers were found for this account.") elif len(args) == 1: containers = self._connection.list_container_object_names(args[0]) if not containers: print("No matching container found.") else: raise CommandError("Pass one and only one [container_name] as an argument") for container in containers: print(container)
def handle(self, *args, **options): if len(args) != 1: raise CommandError( "Pass one and only one [container_name] as an argument") container_name = args[0] if not options.get("is_yes"): is_ok = raw_input( "Permanently delete container {0}? [y|N] ".format( container_name)) if not is_ok == "y": raise CommandError("Aborted") print("Connecting") self._connection = Auth()._get_connection() container = self._connection.get_container(container_name) print("Deleting objects from container {0}".format(container_name)) container.delete_all_objects() container.delete() print("Deletion complete")
def handle(self, *args, **options): if len(args) != 1: raise CommandError("Pass one and only one [container_name] as an argument") container_name = args[0] if not options.get("is_yes"): is_ok = raw_input("Permanently delete container {0}? [y|N] ".format( container_name)) if not is_ok == "y": raise CommandError("Aborted") print("Connecting") self._connection = Auth()._get_connection() container = self._connection.get_container(container_name) print("Deleting objects from container {0}".format(container_name)) container.delete_all_objects() container.delete() print("Deletion complete")
def handle_noargs(self, *args, **options): # setup self.set_options(options) self._connection = Auth()._get_connection() self.container = self._connection.get_container(self.container_name) # wipe first if self.wipe: self.wipe_container() if self.debug: # match local files print "REGION="+CUMULUS["REGION"] abspaths = self.match_local(self.file_root, self.includes, self.excludes) relpaths = [] for path in abspaths: filename = path.split(self.file_root)[1] if filename.startswith("/"): filename = filename[1:] relpaths.append(filename) if not relpaths: settings_root_prefix = "MEDIA" if self.syncmedia else "STATIC" raise CommandError("The {0}_ROOT directory is empty " "or all files have been ignored.".format(settings_root_prefix)) for path in abspaths: if not os.path.isfile(path): raise CommandError("Unsupported filetype: {0}.".format(path)) # match cloud objects cloud_objs = self.match_cloud(self.includes, self.excludes) remote_objects = { obj.name: datetime.datetime.strptime( obj.last_modified, "%Y-%m-%dT%H:%M:%S.%f") for obj in self.container.get_objects() } # sync self.upload_files(abspaths, relpaths, remote_objects) self.delete_extra_files(relpaths, cloud_objs) if not self.quiet or self.verbosity > 1: self.print_tally()
class Command(BaseCommand): help = "Create a container." args = "[container_name]" option_list = BaseCommand.option_list + ( optparse.make_option("-p", "--private", action="store_true", default=False, dest="private", help="Make a private container."),) def handle(self, *args, **options): if len(args) != 1: raise CommandError("Pass one and only one [container_name] as an argument") self._connection = Auth()._get_connection() container_name = args[0] print("Creating container: {0}".format(container_name)) container = self._connection.create_container(container_name) if options.get("private"): print("Private container: {0}".format(container_name)) container.make_private() else: print("Public container: {0}".format(container_name)) container.make_public(ttl=CUMULUS["TTL"])
class Command(BaseCommand): help = "Create a container." args = "[container_name]" def add_arguments(self, parser): parser.add_argument('-p', '--private', action='store_true', default=False, dest='private', help='Assume Yes to confiramtion question') def handle(self, *args, **options): if len(args) != 1: raise CommandError("Pass one and only one [container_name] as an argument") self._connection = Auth()._get_connection() container_name = args[0] print("Creating container: {0}".format(container_name)) container = self._connection.create_container(container_name) if options.get("private"): print("Private container: {0}".format(container_name)) container.make_private() else: print("Public container: {0}".format(container_name)) container.make_public(ttl=CUMULUS["TTL"])
class Command(NoArgsCommand): help = "Synchronizes project static *or* media files to cloud files." option_list = NoArgsCommand.option_list + ( optparse.make_option("-i", "--include", action="append", default=[], dest="includes", metavar="PATTERN", help="Include file or directories matching this glob-style " "pattern. Use multiple times to include more."), optparse.make_option("-e", "--exclude", action="append", default=[], dest="excludes", metavar="PATTERN", help="Exclude files or directories matching this glob-style " "pattern. Use multiple times to exclude more."), optparse.make_option("-w", "--wipe", action="store_true", dest="wipe", default=False, help="Wipes out entire contents of container first."), optparse.make_option("-t", "--test-run", action="store_true", dest="test_run", default=False, help="Performs a test run of the sync."), optparse.make_option("-q", "--quiet", action="store_true", dest="test_run", default=False, help="Do not display any output."), optparse.make_option("-c", "--container", dest="container", help="Override STATIC_CONTAINER."), optparse.make_option("-s", "--static", action="store_true", dest="syncstatic", default=False, help="Sync static files located at settings.STATIC_ROOT path."), optparse.make_option("-m", "--media", action="store_true", dest="syncmedia", default=False, help="Sync media files located at settings.MEDIA_ROOT path."), ) api_key = CUMULUS["API_KEY"] region = CUMULUS["REGION"] use_snet = CUMULUS["SERVICENET"] username = CUMULUS["USERNAME"] def __init__(self, username=None, api_key=None, container=None, connection_kwargs=None, container_uri=None): """ Initializes the settings for the connection and container. """ if username is not None: self.username = username if api_key is not None: self.api_key = api_key # connect if CUMULUS["USE_PYRAX"]: if CUMULUS["PYRAX_IDENTITY_TYPE"]: pyrax.set_setting("identity_type", CUMULUS["PYRAX_IDENTITY_TYPE"]) if CUMULUS["AUTH_URL"]: pyrax.set_setting("auth_endpoint", CUMULUS["AUTH_URL"]) if CUMULUS["AUTH_TENANT_ID"]: pyrax.set_setting("tenant_id", CUMULUS["AUTH_TENANT_ID"]) pyrax.set_credentials(self.username, self.api_key) super(Command, self).__init__() def set_options(self, options): """ Sets instance variables based on an options dict """ # COMMAND LINE OPTIONS self.wipe = options.get("wipe") self.test_run = options.get("test_run") self.quiet = options.get("test_run") self.container_name = options.get("container") self.verbosity = int(options.get("verbosity")) self.syncmedia = options.get("syncmedia") self.syncstatic = options.get("syncstatic") if self.test_run: self.verbosity = 2 cli_includes = options.get("includes") cli_excludes = options.get("excludes") # CUMULUS CONNECTION AND SETTINGS FROM SETTINGS.PY if self.syncmedia and self.syncstatic: raise CommandError("options --media and --static are mutually exclusive") if not self.container_name: if self.syncmedia: self.container_name = CUMULUS["CONTAINER"] elif self.syncstatic: self.container_name = CUMULUS["STATIC_CONTAINER"] else: raise CommandError("must select one of the required options, either --media or --static") settings_includes = CUMULUS["INCLUDE_LIST"] settings_excludes = CUMULUS["EXCLUDE_LIST"] # PATH SETTINGS if self.syncmedia: self.file_root = os.path.abspath(settings.MEDIA_ROOT) self.file_url = settings.MEDIA_URL elif self.syncstatic: self.file_root = os.path.abspath(settings.STATIC_ROOT) self.file_url = settings.STATIC_URL if not self.file_root.endswith("/"): self.file_root = self.file_root + "/" if self.file_url.startswith("/"): self.file_url = self.file_url[1:] # SYNCSTATIC VARS # combine includes and excludes from the cli and django settings file self.includes = list(set(cli_includes + settings_includes)) self.excludes = list(set(cli_excludes + settings_excludes)) # transform glob patterns to regular expressions self.local_filenames = [] self.create_count = 0 self.upload_count = 0 self.update_count = 0 self.skip_count = 0 self.delete_count = 0 def connect_container(self): """ Connects to a container using the swiftclient api. The container will be created and/or made public using the pyrax api if not already so. """ if CUMULUS["USE_PYRAX"]: public = not self.use_snet # invert self.conn = pyrax.connect_to_cloudfiles(region=self.region, public=public) else: self.conn = swiftclient.Connection(authurl=CUMULUS["AUTH_URL"], user=CUMULUS["USERNAME"], key=CUMULUS["API_KEY"], snet=CUMULUS["SERVICENET"], auth_version=CUMULUS["AUTH_VERSION"], tenant_name=CUMULUS["AUTH_TENANT_NAME"]) #try: # self.conn.head_container(self.container_name) #except swiftclient.client.ClientException as exception: # if exception.msg == "Container HEAD failed": # call_command("container_create", self.container_name) # else: # raise self.container = self.conn.get_container(self.container_name) def handle_noargs(self, *args, **options): # setup self.set_options(options) self._connection = Auth()._get_connection() self.container = self._connection.get_container(self.container_name) # wipe first if self.wipe: self.wipe_container() # match local files abspaths = self.match_local(self.file_root, self.includes, self.excludes) relpaths = [] for path in abspaths: filename = path.split(self.file_root)[1] if filename.startswith("/"): filename = filename[1:] relpaths.append(filename) if not relpaths: settings_root_prefix = "MEDIA" if self.syncmedia else "STATIC" raise CommandError("The {0}_ROOT directory is empty " "or all files have been ignored.".format(settings_root_prefix)) for path in abspaths: if not os.path.isfile(path): raise CommandError("Unsupported filetype: {0}.".format(path)) # match cloud objects cloud_objs = self.match_cloud(self.includes, self.excludes) remote_objects = { obj.name: datetime.datetime.strptime( obj.last_modified, "%Y-%m-%dT%H:%M:%S.%f") for obj in self.container.get_objects() } # sync self.upload_files(abspaths, relpaths, remote_objects) self.delete_extra_files(relpaths, cloud_objs) if not self.quiet or self.verbosity > 1: self.print_tally() def match_cloud(self, includes, excludes): """ Returns the cloud objects that match the include and exclude patterns. """ cloud_objs = [cloud_obj.name for cloud_obj in self.container.get_objects()] includes_pattern = r"|".join([fnmatch.translate(x) for x in includes]) excludes_pattern = r"|".join([fnmatch.translate(x) for x in excludes]) or r"$." excludes = [o for o in cloud_objs if re.match(excludes_pattern, o)] includes = [o for o in cloud_objs if re.match(includes_pattern, o)] return [o for o in includes if o not in excludes] def match_local(self, prefix, includes, excludes): """ Filters os.walk() with include and exclude patterns. See: http://stackoverflow.com/a/5141829/93559 """ includes_pattern = r"|".join([fnmatch.translate(x) for x in includes]) excludes_pattern = r"|".join([fnmatch.translate(x) for x in excludes]) or r"$." matches = [] for root, dirs, files in os.walk(prefix, topdown=True): # exclude dirs dirs[:] = [os.path.join(root, d) for d in dirs] dirs[:] = [d for d in dirs if not re.match(excludes_pattern, d.split(root)[1])] # exclude/include files files = [os.path.join(root, f) for f in files] files = [os.path.join(root, f) for f in files if not re.match(excludes_pattern, f)] files = [os.path.join(root, f) for f in files if re.match(includes_pattern, f.split(prefix)[1])] for fname in files: matches.append(fname) return matches def upload_files(self, abspaths, relpaths, remote_objects): """ Determines files to be uploaded and call ``upload_file`` on each. """ for relpath in relpaths: abspath = [p for p in abspaths if p.endswith(relpath)][0] cloud_datetime = remote_objects[relpath] if relpath in remote_objects else None local_datetime = datetime.datetime.utcfromtimestamp(os.stat(abspath).st_mtime) if cloud_datetime and local_datetime < cloud_datetime: self.skip_count += 1 if not self.quiet: print("Skipped {0}: not modified.".format(relpath)) continue if relpath in remote_objects: self.update_count += 1 else: self.create_count += 1 self.upload_file(abspath, relpath) def upload_file(self, abspath, cloud_filename): """ Uploads a file to the container. """ if not self.test_run: content = open(abspath, "rb") content_type = get_content_type(cloud_filename, content) headers = get_headers(cloud_filename, content_type) if headers.get("Content-Encoding") == "gzip": content = get_gzipped_contents(content) size = content.size else: size = os.stat(abspath).st_size self.container.create( obj_name=cloud_filename, data=content, content_type=content_type, content_length=size, content_encoding=headers.get("Content-Encoding", None), headers=headers, ttl=CUMULUS["FILE_TTL"], etag=None, ) self.upload_count += 1 if not self.quiet or self.verbosity > 1: print("Uploaded: {0}".format(cloud_filename)) def delete_extra_files(self, relpaths, cloud_objs): """ Deletes any objects from the container that do not exist locally. """ for cloud_obj in cloud_objs: if cloud_obj not in relpaths: if not self.test_run: self.delete_cloud_obj(cloud_obj) self.delete_count += 1 if not self.quiet or self.verbosity > 1: print("Deleted: {0}".format(cloud_obj)) def delete_cloud_obj(self, cloud_obj): """ Deletes an object from the container. """ self._connection.delete_object( container=self.container_name, obj=cloud_obj, ) def wipe_container(self): """ Completely wipes out the contents of the container. """ if self.test_run: print("Wipe would delete {0} objects.".format(len(self.container.object_count))) else: if not self.quiet or self.verbosity > 1: print("Deleting {0} objects...".format(len(self.container.object_count))) self._connection.delete_all_objects() def print_tally(self): """ Prints the final tally to stdout. """ self.update_count = self.upload_count - self.create_count if self.test_run: print("Test run complete with the following results:") print("Skipped {0}. Created {1}. Updated {2}. Deleted {3}.".format( self.skip_count, self.create_count, self.update_count, self.delete_count))
class Command(NoArgsCommand): help = "Synchronizes project static *or* media files to cloud files." option_list = NoArgsCommand.option_list + ( optparse.make_option("-i", "--include", action="append", default=[], dest="includes", metavar="PATTERN", help="Include file or directories matching this glob-style " "pattern. Use multiple times to include more."), optparse.make_option("-e", "--exclude", action="append", default=[], dest="excludes", metavar="PATTERN", help="Exclude files or directories matching this glob-style " "pattern. Use multiple times to exclude more."), optparse.make_option("-w", "--wipe", action="store_true", dest="wipe", default=False, help="Wipes out entire contents of container first."), optparse.make_option("-t", "--test-run", action="store_true", dest="test_run", default=False, help="Performs a test run of the sync."), optparse.make_option("-q", "--quiet", action="store_true", dest="test_run", default=False, help="Do not display any output."), optparse.make_option("-c", "--container", dest="container", help="Override STATIC_CONTAINER."), optparse.make_option("-s", "--static", action="store_true", dest="syncstatic", default=False, help="Sync static files located at settings.STATIC_ROOT path."), optparse.make_option("-m", "--media", action="store_true", dest="syncmedia", default=False, help="Sync media files located at settings.MEDIA_ROOT path."), ) def set_options(self, options): """ Sets instance variables based on an options dict """ # COMMAND LINE OPTIONS self.wipe = options.get("wipe") self.test_run = options.get("test_run") self.quiet = options.get("test_run") self.container_name = options.get("container") self.verbosity = int(options.get("verbosity")) self.syncmedia = options.get("syncmedia") self.syncstatic = options.get("syncstatic") if self.test_run: self.verbosity = 2 cli_includes = options.get("includes") cli_excludes = options.get("excludes") # CUMULUS CONNECTION AND SETTINGS FROM SETTINGS.PY if self.syncmedia and self.syncstatic: raise CommandError("options --media and --static are mutually exclusive") if not self.container_name: if self.syncmedia: self.container_name = CUMULUS["CONTAINER"] elif self.syncstatic: self.container_name = CUMULUS["STATIC_CONTAINER"] else: raise CommandError("must select one of the required options, either --media or --static") settings_includes = CUMULUS["INCLUDE_LIST"] settings_excludes = CUMULUS["EXCLUDE_LIST"] # PATH SETTINGS if self.syncmedia: self.file_root = os.path.abspath(settings.MEDIA_ROOT) self.file_url = settings.MEDIA_URL elif self.syncstatic: self.file_root = os.path.abspath(settings.STATIC_ROOT) self.file_url = settings.STATIC_URL if not self.file_root.endswith("/"): self.file_root = self.file_root + "/" if self.file_url.startswith("/"): self.file_url = self.file_url[1:] # SYNCSTATIC VARS # combine includes and excludes from the cli and django settings file self.includes = list(set(cli_includes + settings_includes)) self.excludes = list(set(cli_excludes + settings_excludes)) # transform glob patterns to regular expressions self.local_filenames = [] self.create_count = 0 self.upload_count = 0 self.update_count = 0 self.skip_count = 0 self.delete_count = 0 def handle_noargs(self, *args, **options): # setup self.set_options(options) self._connection = Auth()._get_connection() self.container = self._connection.get_container(self.container_name) # wipe first if self.wipe: self.wipe_container() # match local files abspaths = self.match_local(self.file_root, self.includes, self.excludes) relpaths = [] for path in abspaths: filename = path.split(self.file_root)[1] if filename.startswith("/"): filename = filename[1:] relpaths.append(filename) if not relpaths: settings_root_prefix = "MEDIA" if self.syncmedia else "STATIC" raise CommandError("The {0}_ROOT directory is empty " "or all files have been ignored.".format(settings_root_prefix)) for path in abspaths: if not os.path.isfile(path): raise CommandError("Unsupported filetype: {0}.".format(path)) # match cloud objects cloud_objs = self.match_cloud(self.includes, self.excludes) remote_objects = { obj.name: datetime.datetime.strptime( obj.last_modified, "%Y-%m-%dT%H:%M:%S.%f") for obj in self.container.get_objects() } # sync self.upload_files(abspaths, relpaths, remote_objects) self.delete_extra_files(relpaths, cloud_objs) if not self.quiet or self.verbosity > 1: self.print_tally() def match_cloud(self, includes, excludes): """ Returns the cloud objects that match the include and exclude patterns. """ cloud_objs = [cloud_obj.name for cloud_obj in self.container.get_objects()] includes_pattern = r"|".join([fnmatch.translate(x) for x in includes]) excludes_pattern = r"|".join([fnmatch.translate(x) for x in excludes]) or r"$." excludes = [o for o in cloud_objs if re.match(excludes_pattern, o)] includes = [o for o in cloud_objs if re.match(includes_pattern, o)] return [o for o in includes if o not in excludes] def match_local(self, prefix, includes, excludes): """ Filters os.walk() with include and exclude patterns. See: http://stackoverflow.com/a/5141829/93559 """ includes_pattern = r"|".join([fnmatch.translate(x) for x in includes]) excludes_pattern = r"|".join([fnmatch.translate(x) for x in excludes]) or r"$." matches = [] for root, dirs, files in os.walk(prefix, topdown=True): # exclude dirs dirs[:] = [os.path.join(root, d) for d in dirs] dirs[:] = [d for d in dirs if not re.match(excludes_pattern, d.split(root)[1])] # exclude/include files files = [os.path.join(root, f) for f in files] files = [os.path.join(root, f) for f in files if not re.match(excludes_pattern, f)] files = [os.path.join(root, f) for f in files if re.match(includes_pattern, f.split(prefix)[1])] for fname in files: matches.append(fname) return matches def upload_files(self, abspaths, relpaths, remote_objects): """ Determines files to be uploaded and call ``upload_file`` on each. """ for relpath in relpaths: abspath = [p for p in abspaths if p[len(self.file_root):] == relpath][0] cloud_datetime = remote_objects[relpath] if relpath in remote_objects else None local_datetime = datetime.datetime.utcfromtimestamp(os.stat(abspath).st_mtime) if cloud_datetime and local_datetime < cloud_datetime: self.skip_count += 1 if not self.quiet: print("Skipped {0}: not modified.".format(relpath)) continue if relpath in remote_objects: self.update_count += 1 else: self.create_count += 1 self.upload_file(abspath, relpath) def upload_file(self, abspath, cloud_filename): """ Uploads a file to the container. """ if not self.test_run: content = open(abspath, "rb") content_type = get_content_type(cloud_filename, content) headers = get_headers(cloud_filename, content_type) if headers.get("Content-Encoding") == "gzip": content = get_gzipped_contents(content) size = content.size else: size = os.stat(abspath).st_size self.container.create( obj_name=cloud_filename, data=content, content_type=content_type, content_length=size, content_encoding=headers.get("Content-Encoding", None), headers=headers, ttl=CUMULUS["FILE_TTL"], etag=None, ) self.upload_count += 1 if not self.quiet or self.verbosity > 1: print("Uploaded: {0}".format(cloud_filename)) def delete_extra_files(self, relpaths, cloud_objs): """ Deletes any objects from the container that do not exist locally. """ for cloud_obj in cloud_objs: if cloud_obj not in relpaths: if not self.test_run: self.delete_cloud_obj(cloud_obj) self.delete_count += 1 if not self.quiet or self.verbosity > 1: print("Deleted: {0}".format(cloud_obj)) def delete_cloud_obj(self, cloud_obj): """ Deletes an object from the container. """ self._connection.delete_object( container=self.container_name, obj=cloud_obj, ) def wipe_container(self): """ Completely wipes out the contents of the container. """ if self.test_run: print("Wipe would delete {0} objects.".format(len(self.container.object_count))) else: if not self.quiet or self.verbosity > 1: print("Deleting {0} objects...".format(len(self.container.object_count))) self._connection.delete_all_objects() def print_tally(self): """ Prints the final tally to stdout. """ self.update_count = self.upload_count - self.create_count if self.test_run: print("Test run complete with the following results:") print("Skipped {0}. Created {1}. Updated {2}. Deleted {3}.".format( self.skip_count, self.create_count, self.update_count, self.delete_count))
class Command(CollectStaticCommand): """ Command that allows to copy or symlink media files from Rackspace Cloud Files location to the settings.MEDIA_ROOT. """ help = "Collect media files in a single location." def __init__(self, *args, **kwargs): super(Command, self).__init__(*args, **kwargs) self.copied_files = [] self.symlinked_files = [] self.unmodified_files = [] self.post_processed_files = [] self.storage = swiftclient_storage self.style = no_style() try: self.storage.path('') except NotImplementedError: self.local = False else: self.local = True def set_options(self, **options): """ Set instance variables based on an options dict """ self.interactive = options['interactive'] self.verbosity = options['verbosity'] self.symlink = options['link'] self.clear = options['clear'] self.dry_run = options['dry_run'] ignore_patterns = options['ignore_patterns'] if options['use_default_ignore_patterns']: ignore_patterns += ['CVS', '.*', '*~'] self.ignore_patterns = list(set(ignore_patterns)) self.post_process = options['post_process'] self.container_name = CUMULUS["CONTAINER"] self._connection = Auth()._get_connection() self.container = self._connection.get_container(self.container_name) def collect(self): """ Perform the bulk of the work of collectstatic. Split off from handle() to facilitate testing. """ if self.symlink and not self.local: raise CommandError("Can't symlink to a remote destination.") if self.clear: self.clear_dir('') if self.symlink: handler = self.link_file else: handler = self.copy_file cloud_objects = self.container.get_objects() for obj in cloud_objects: if self.dry_run: self.log("Pretending to copy '%s'" % obj.name, level=1) else: self.log("Copying '%s'" % obj.name, level=1) obj.download(settings.MEDIA_ROOT) # TODO: don't overwrite files with matching timestamps # TODO: delete local files that don't match any remote objects return { 'modified': [obj.name for obj in cloud_objects], 'unmodified': [], 'post_processed': [], }