class Command(BaseCommand):
help = ("List all the items in a container to stdout.\n\n"
"We recommend you run it like this:\n"
" ./manage.py container_list <container> | pv --line-mode > <container>.list\n\n"
"pv is Pipe Viewer: http://www.ivarch.com/programs/pv.shtml")
args = "[container_name]"
def handle(self, *args, **options):
"""
Lists all the items in a container to stdout.
"""
self._connection = Auth()._get_connection()
if len(args) == 0:
containers = self._connection.list_containers()
if not containers:
print("No containers were found for this account.")
elif len(args) == 1:
containers = self._connection.list_container_object_names(args[0])
if not containers:
print("No matching container found.")
else:
raise CommandError("Pass one and only one [container_name] as an argument")
for container in containers:
print(container)
class Command(BaseCommand):
help = "Display info for containers"
args = "[container_name container_name ...]"
option_list = BaseCommand.option_list + (
optparse.make_option(
"-n", "--name", action="store_true", dest="name", default=False),
optparse.make_option(
"-c", "--count", action="store_true", dest="count", default=False),
optparse.make_option(
"-s", "--size", action="store_true", dest="size", default=False),
optparse.make_option(
"-u", "--uri", action="store_true", dest="uri", default=False))
def handle(self, *args, **options):
self._connection = Auth()._get_connection()
container_names = self._connection.list_container_names()
if args:
matches = []
for container_name in container_names:
if container_name in args:
matches.append(container_name)
container_names = matches
if not container_names:
print("No containers found.")
return
if not args:
account_details = self._connection.get_account_details()
print("container_count | object_count | bytes_used")
print("{0}, {1}, {2}\n".format(
account_details["container_count"],
account_details["object_count"],
account_details["bytes_used"],
))
opts = ["name", "count", "size", "uri"]
output = [o for o in opts if options.get(o)]
if output:
print(" | ".join(output))
else:
print(" | ".join(opts))
for container_name in container_names:
container = self._connection.get_container(container_name)
info = {
"name": container.name,
"count": container.object_count,
"size": container.total_bytes,
"cdn_enabled": container.cdn_enabled,
"uri": container.cdn_uri if container.cdn_enabled else None,
}
output = [str(info[o]) for o in opts if options.get(o)]
if not output:
output = [str(info[o]) for o in opts]
print(", ".join(output))
class Command(BaseCommand):
help = "Display info for containers"
args = "[container_name container_name ...]"
option_list = BaseCommand.option_list + (
optparse.make_option("-n", "--name", action="store_true", dest="name", default=False),
optparse.make_option("-c", "--count", action="store_true", dest="count", default=False),
optparse.make_option("-s", "--size", action="store_true", dest="size", default=False),
optparse.make_option("-u", "--uri", action="store_true", dest="uri", default=False)
)
def handle(self, *args, **options):
self._connection = Auth()._get_connection()
container_names = self._connection.list_container_names()
if args:
matches = []
for container_name in container_names:
if container_name in args:
matches.append(container_name)
container_names = matches
if not container_names:
print("No containers found.")
return
if not args:
account_details = self._connection.get_account_details()
print("container_count | object_count | bytes_used")
print("{0}, {1}, {2}\n".format(
account_details["container_count"],
account_details["object_count"],
account_details["bytes_used"],
))
opts = ["name", "count", "size", "uri"]
output = [o for o in opts if options.get(o)]
if output:
print(" | ".join(output))
else:
print(" | ".join(opts))
for container_name in container_names:
container = self._connection.get_container(container_name)
info = {
"name": container.name,
"count": container.object_count,
"size": container.total_bytes,
"cdn_enabled": container.cdn_enabled,
"uri": container.cdn_uri if container.cdn_enabled else None,
}
output = [str(info[o]) for o in opts if options.get(o)]
if not output:
output = [str(info[o]) for o in opts]
print(", ".join(output))
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Pass one and only one [container_name] as an argument")
self._connection = Auth()._get_connection()
container_name = args[0]
print("Creating container: {0}".format(container_name))
container = self._connection.create_container(container_name)
if options.get("private"):
print("Private container: {0}".format(container_name))
container.make_private()
else:
print("Public container: {0}".format(container_name))
container.make_public(ttl=CUMULUS["TTL"])
class Command(BaseCommand):
help = "Delete a container."
args = "[container_name]"
def add_arguments(self, parser):
parser.add_argument('-y', '--yes', action='store_true', default=False,
dest='is_yes', help='Assume Yes to confiramtion question')
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Pass one and only one [container_name] as an argument")
container_name = args[0]
if not options.get("is_yes"):
is_ok = raw_input("Permanently delete container {0}? [y|N] ".format(
container_name))
if not is_ok == "y":
raise CommandError("Aborted")
print("Connecting")
self._connection = Auth()._get_connection()
container = self._connection.get_container(container_name)
print("Deleting objects from container {0}".format(container_name))
container.delete_all_objects()
container.delete()
print("Deletion complete")
class Command(BaseCommand):
help = "Delete a container."
args = "[container_name]"
option_list = BaseCommand.option_list + (optparse.make_option(
"-y",
"--yes",
action="store_true",
default=False,
dest="is_yes",
help="Assume Yes to confirmation question"), )
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError(
"Pass one and only one [container_name] as an argument")
container_name = args[0]
if not options.get("is_yes"):
is_ok = raw_input(
"Permanently delete container {0}? [y|N] ".format(
container_name))
if not is_ok == "y":
raise CommandError("Aborted")
print("Connecting")
self._connection = Auth()._get_connection()
container = self._connection.get_container(container_name)
print("Deleting objects from container {0}".format(container_name))
container.delete_all_objects()
container.delete()
print("Deletion complete")
class Command(BaseCommand):
help = "Delete a container."
args = "[container_name]"
option_list = BaseCommand.option_list + (
optparse.make_option("-y", "--yes", action="store_true", default=False,
dest="is_yes", help="Assume Yes to confirmation question"),)
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Pass one and only one [container_name] as an argument")
container_name = args[0]
if not options.get("is_yes"):
is_ok = raw_input("Permanently delete container {0}? [y|N] ".format(
container_name))
if not is_ok == "y":
raise CommandError("Aborted")
print("Connecting")
self._connection = Auth()._get_connection()
container = self._connection.get_container(container_name)
print("Deleting objects from container {0}".format(container_name))
container.delete_all_objects()
container.delete()
print("Deletion complete")
class Command(BaseCommand):
help = "Delete a container."
args = "[container_name]"
def add_arguments(self, parser):
parser.add_argument('-y',
'--yes',
action='store_true',
default=False,
dest='is_yes',
help='Assume Yes to confiramtion question')
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError(
"Pass one and only one [container_name] as an argument")
container_name = args[0]
if not options.get("is_yes"):
is_ok = raw_input(
"Permanently delete container {0}? [y|N] ".format(
container_name))
if not is_ok == "y":
raise CommandError("Aborted")
print("Connecting")
self._connection = Auth()._get_connection()
container = self._connection.get_container(container_name)
print("Deleting objects from container {0}".format(container_name))
container.delete_all_objects()
container.delete()
print("Deletion complete")
class Command(BaseCommand):
help = "Create a container."
args = "[container_name]"
def add_arguments(self, parser):
parser.add_argument('-p',
'--private',
action='store_true',
default=False,
dest='private',
help='Assume Yes to confiramtion question')
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError(
"Pass one and only one [container_name] as an argument")
self._connection = Auth()._get_connection()
container_name = args[0]
print("Creating container: {0}".format(container_name))
container = self._connection.create_container(container_name)
if options.get("private"):
print("Private container: {0}".format(container_name))
container.make_private()
else:
print("Public container: {0}".format(container_name))
container.make_public(ttl=CUMULUS["TTL"])
def handle(self, *args, **options):
self._connection = Auth()._get_connection()
container_names = self._connection.list_container_names()
if args:
matches = []
for container_name in container_names:
if container_name in args:
matches.append(container_name)
container_names = matches
if not container_names:
print("No containers found.")
return
if not args:
account_details = self._connection.get_account_details()
print("container_count | object_count | bytes_used")
print("{0}, {1}, {2}\n".format(
account_details["container_count"],
account_details["object_count"],
account_details["bytes_used"],
))
opts = ["name", "count", "size", "uri"]
output = [o for o in opts if options.get(o)]
if output:
print(" | ".join(output))
else:
print(" | ".join(opts))
for container_name in container_names:
container = self._connection.get_container(container_name)
info = {
"name": container.name,
"count": container.object_count,
"size": container.total_bytes,
"cdn_enabled": container.cdn_enabled,
"uri": container.cdn_uri if container.cdn_enabled else None,
}
output = [str(info[o]) for o in opts if options.get(o)]
if not output:
output = [str(info[o]) for o in opts]
print(", ".join(output))
def handle_noargs(self, *args, **options):
# setup
self.set_options(options)
self._connection = Auth()._get_connection()
self.container = self._connection.get_container(self.container_name)
# wipe first
if self.wipe:
self.wipe_container()
# match local files
abspaths = self.match_local(self.file_root, self.includes,
self.excludes)
relpaths = []
for path in abspaths:
filename = path.split(self.file_root)[1]
if filename.startswith("/"):
filename = filename[1:]
relpaths.append(filename)
if not relpaths:
settings_root_prefix = "MEDIA" if self.syncmedia else "STATIC"
raise CommandError(
"The {0}_ROOT directory is empty "
"or all files have been ignored.".format(settings_root_prefix))
for path in abspaths:
if not os.path.isfile(path):
raise CommandError("Unsupported filetype: {0}.".format(path))
# match cloud objects
cloud_objs = self.match_cloud(self.includes, self.excludes)
remote_objects = {
obj.name: datetime.datetime.strptime(obj.last_modified,
"%Y-%m-%dT%H:%M:%S.%f")
for obj in self.container.get_objects()
}
# sync
self.upload_files(abspaths, relpaths, remote_objects)
self.delete_extra_files(relpaths, cloud_objs)
if not self.quiet or self.verbosity > 1:
self.print_tally()
def set_options(self, **options):
"""
Set instance variables based on an options dict
"""
self.interactive = options['interactive']
self.verbosity = options['verbosity']
self.symlink = options['link']
self.clear = options['clear']
self.dry_run = options['dry_run']
ignore_patterns = options['ignore_patterns']
if options['use_default_ignore_patterns']:
ignore_patterns += ['CVS', '.*', '*~']
self.ignore_patterns = list(set(ignore_patterns))
self.post_process = options['post_process']
self.container_name = CUMULUS["CONTAINER"]
self._connection = Auth()._get_connection()
self.container = self._connection.get_container(self.container_name)
def handle(self, *args, **options):
"""
Lists all the items in a container to stdout.
"""
self._connection = Auth()._get_connection()
if len(args) == 0:
containers = self._connection.list_containers()
if not containers:
print("No containers were found for this account.")
elif len(args) == 1:
containers = self._connection.list_container_object_names(args[0])
if not containers:
print("No matching container found.")
else:
raise CommandError("Pass one and only one [container_name] as an argument")
for container in containers:
print(container)
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError(
"Pass one and only one [container_name] as an argument")
container_name = args[0]
if not options.get("is_yes"):
is_ok = raw_input(
"Permanently delete container {0}? [y|N] ".format(
container_name))
if not is_ok == "y":
raise CommandError("Aborted")
print("Connecting")
self._connection = Auth()._get_connection()
container = self._connection.get_container(container_name)
print("Deleting objects from container {0}".format(container_name))
container.delete_all_objects()
container.delete()
print("Deletion complete")
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Pass one and only one [container_name] as an argument")
container_name = args[0]
if not options.get("is_yes"):
is_ok = raw_input("Permanently delete container {0}? [y|N] ".format(
container_name))
if not is_ok == "y":
raise CommandError("Aborted")
print("Connecting")
self._connection = Auth()._get_connection()
container = self._connection.get_container(container_name)
print("Deleting objects from container {0}".format(container_name))
container.delete_all_objects()
container.delete()
print("Deletion complete")
def handle(self, *args, **options):
self._connection = Auth()._get_connection()
container_names = self._connection.list_container_names()
if args:
matches = []
for container_name in container_names:
if container_name in args:
matches.append(container_name)
container_names = matches
if not container_names:
print("No containers found.")
return
if not args:
account_details = self._connection.get_account_details()
print("container_count | object_count | bytes_used")
print("{0}, {1}, {2}\n".format(
account_details["container_count"],
account_details["object_count"],
account_details["bytes_used"],
))
opts = ["name", "count", "size", "uri"]
output = [o for o in opts if options.get(o)]
if output:
print(" | ".join(output))
else:
print(" | ".join(opts))
for container_name in container_names:
container = self._connection.get_container(container_name)
info = {
"name": container.name,
"count": container.object_count,
"size": container.total_bytes,
"cdn_enabled": container.cdn_enabled,
"uri": container.cdn_uri if container.cdn_enabled else None,
}
output = [str(info[o]) for o in opts if options.get(o)]
if not output:
output = [str(info[o]) for o in opts]
print(", ".join(output))
def handle_noargs(self, *args, **options):
# setup
self.set_options(options)
self._connection = Auth()._get_connection()
self.container = self._connection.get_container(self.container_name)
# wipe first
if self.wipe:
self.wipe_container()
if self.debug:
# match local files
print "REGION="+CUMULUS["REGION"]
abspaths = self.match_local(self.file_root, self.includes, self.excludes)
relpaths = []
for path in abspaths:
filename = path.split(self.file_root)[1]
if filename.startswith("/"):
filename = filename[1:]
relpaths.append(filename)
if not relpaths:
settings_root_prefix = "MEDIA" if self.syncmedia else "STATIC"
raise CommandError("The {0}_ROOT directory is empty "
"or all files have been ignored.".format(settings_root_prefix))
for path in abspaths:
if not os.path.isfile(path):
raise CommandError("Unsupported filetype: {0}.".format(path))
# match cloud objects
cloud_objs = self.match_cloud(self.includes, self.excludes)
remote_objects = {
obj.name: datetime.datetime.strptime(
obj.last_modified,
"%Y-%m-%dT%H:%M:%S.%f") for obj in self.container.get_objects()
}
# sync
self.upload_files(abspaths, relpaths, remote_objects)
self.delete_extra_files(relpaths, cloud_objs)
if not self.quiet or self.verbosity > 1:
self.print_tally()
class Command(BaseCommand):
help = "Create a container."
args = "[container_name]"
option_list = BaseCommand.option_list + (
optparse.make_option("-p", "--private", action="store_true", default=False,
dest="private", help="Make a private container."),)
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Pass one and only one [container_name] as an argument")
self._connection = Auth()._get_connection()
container_name = args[0]
print("Creating container: {0}".format(container_name))
container = self._connection.create_container(container_name)
if options.get("private"):
print("Private container: {0}".format(container_name))
container.make_private()
else:
print("Public container: {0}".format(container_name))
container.make_public(ttl=CUMULUS["TTL"])
class Command(BaseCommand):
help = "Create a container."
args = "[container_name]"
def add_arguments(self, parser):
parser.add_argument('-p', '--private', action='store_true', default=False,
dest='private', help='Assume Yes to confiramtion question')
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Pass one and only one [container_name] as an argument")
self._connection = Auth()._get_connection()
container_name = args[0]
print("Creating container: {0}".format(container_name))
container = self._connection.create_container(container_name)
if options.get("private"):
print("Private container: {0}".format(container_name))
container.make_private()
else:
print("Public container: {0}".format(container_name))
container.make_public(ttl=CUMULUS["TTL"])
class Command(NoArgsCommand):
help = "Synchronizes project static *or* media files to cloud files."
option_list = NoArgsCommand.option_list + (
optparse.make_option("-i", "--include", action="append", default=[],
dest="includes", metavar="PATTERN",
help="Include file or directories matching this glob-style "
"pattern. Use multiple times to include more."),
optparse.make_option("-e", "--exclude", action="append", default=[],
dest="excludes", metavar="PATTERN",
help="Exclude files or directories matching this glob-style "
"pattern. Use multiple times to exclude more."),
optparse.make_option("-w", "--wipe",
action="store_true", dest="wipe", default=False,
help="Wipes out entire contents of container first."),
optparse.make_option("-t", "--test-run",
action="store_true", dest="test_run", default=False,
help="Performs a test run of the sync."),
optparse.make_option("-q", "--quiet",
action="store_true", dest="test_run", default=False,
help="Do not display any output."),
optparse.make_option("-c", "--container",
dest="container", help="Override STATIC_CONTAINER."),
optparse.make_option("-s", "--static",
action="store_true", dest="syncstatic", default=False,
help="Sync static files located at settings.STATIC_ROOT path."),
optparse.make_option("-m", "--media",
action="store_true", dest="syncmedia", default=False,
help="Sync media files located at settings.MEDIA_ROOT path."),
)
api_key = CUMULUS["API_KEY"]
region = CUMULUS["REGION"]
use_snet = CUMULUS["SERVICENET"]
username = CUMULUS["USERNAME"]
def __init__(self, username=None, api_key=None, container=None,
connection_kwargs=None, container_uri=None):
"""
Initializes the settings for the connection and container.
"""
if username is not None:
self.username = username
if api_key is not None:
self.api_key = api_key
# connect
if CUMULUS["USE_PYRAX"]:
if CUMULUS["PYRAX_IDENTITY_TYPE"]:
pyrax.set_setting("identity_type", CUMULUS["PYRAX_IDENTITY_TYPE"])
if CUMULUS["AUTH_URL"]:
pyrax.set_setting("auth_endpoint", CUMULUS["AUTH_URL"])
if CUMULUS["AUTH_TENANT_ID"]:
pyrax.set_setting("tenant_id", CUMULUS["AUTH_TENANT_ID"])
pyrax.set_credentials(self.username, self.api_key)
super(Command, self).__init__()
def set_options(self, options):
"""
Sets instance variables based on an options dict
"""
# COMMAND LINE OPTIONS
self.wipe = options.get("wipe")
self.test_run = options.get("test_run")
self.quiet = options.get("test_run")
self.container_name = options.get("container")
self.verbosity = int(options.get("verbosity"))
self.syncmedia = options.get("syncmedia")
self.syncstatic = options.get("syncstatic")
if self.test_run:
self.verbosity = 2
cli_includes = options.get("includes")
cli_excludes = options.get("excludes")
# CUMULUS CONNECTION AND SETTINGS FROM SETTINGS.PY
if self.syncmedia and self.syncstatic:
raise CommandError("options --media and --static are mutually exclusive")
if not self.container_name:
if self.syncmedia:
self.container_name = CUMULUS["CONTAINER"]
elif self.syncstatic:
self.container_name = CUMULUS["STATIC_CONTAINER"]
else:
raise CommandError("must select one of the required options, either --media or --static")
settings_includes = CUMULUS["INCLUDE_LIST"]
settings_excludes = CUMULUS["EXCLUDE_LIST"]
# PATH SETTINGS
if self.syncmedia:
self.file_root = os.path.abspath(settings.MEDIA_ROOT)
self.file_url = settings.MEDIA_URL
elif self.syncstatic:
self.file_root = os.path.abspath(settings.STATIC_ROOT)
self.file_url = settings.STATIC_URL
if not self.file_root.endswith("/"):
self.file_root = self.file_root + "/"
if self.file_url.startswith("/"):
self.file_url = self.file_url[1:]
# SYNCSTATIC VARS
# combine includes and excludes from the cli and django settings file
self.includes = list(set(cli_includes + settings_includes))
self.excludes = list(set(cli_excludes + settings_excludes))
# transform glob patterns to regular expressions
self.local_filenames = []
self.create_count = 0
self.upload_count = 0
self.update_count = 0
self.skip_count = 0
self.delete_count = 0
def connect_container(self):
"""
Connects to a container using the swiftclient api.
The container will be created and/or made public using the
pyrax api if not already so.
"""
if CUMULUS["USE_PYRAX"]:
public = not self.use_snet # invert
self.conn = pyrax.connect_to_cloudfiles(region=self.region,
public=public)
else:
self.conn = swiftclient.Connection(authurl=CUMULUS["AUTH_URL"],
user=CUMULUS["USERNAME"],
key=CUMULUS["API_KEY"],
snet=CUMULUS["SERVICENET"],
auth_version=CUMULUS["AUTH_VERSION"],
tenant_name=CUMULUS["AUTH_TENANT_NAME"])
#try:
# self.conn.head_container(self.container_name)
#except swiftclient.client.ClientException as exception:
# if exception.msg == "Container HEAD failed":
# call_command("container_create", self.container_name)
# else:
# raise
self.container = self.conn.get_container(self.container_name)
def handle_noargs(self, *args, **options):
# setup
self.set_options(options)
self._connection = Auth()._get_connection()
self.container = self._connection.get_container(self.container_name)
# wipe first
if self.wipe:
self.wipe_container()
# match local files
abspaths = self.match_local(self.file_root, self.includes, self.excludes)
relpaths = []
for path in abspaths:
filename = path.split(self.file_root)[1]
if filename.startswith("/"):
filename = filename[1:]
relpaths.append(filename)
if not relpaths:
settings_root_prefix = "MEDIA" if self.syncmedia else "STATIC"
raise CommandError("The {0}_ROOT directory is empty "
"or all files have been ignored.".format(settings_root_prefix))
for path in abspaths:
if not os.path.isfile(path):
raise CommandError("Unsupported filetype: {0}.".format(path))
# match cloud objects
cloud_objs = self.match_cloud(self.includes, self.excludes)
remote_objects = {
obj.name: datetime.datetime.strptime(
obj.last_modified,
"%Y-%m-%dT%H:%M:%S.%f") for obj in self.container.get_objects()
}
# sync
self.upload_files(abspaths, relpaths, remote_objects)
self.delete_extra_files(relpaths, cloud_objs)
if not self.quiet or self.verbosity > 1:
self.print_tally()
def match_cloud(self, includes, excludes):
"""
Returns the cloud objects that match the include and exclude patterns.
"""
cloud_objs = [cloud_obj.name for cloud_obj in self.container.get_objects()]
includes_pattern = r"|".join([fnmatch.translate(x) for x in includes])
excludes_pattern = r"|".join([fnmatch.translate(x) for x in excludes]) or r"$."
excludes = [o for o in cloud_objs if re.match(excludes_pattern, o)]
includes = [o for o in cloud_objs if re.match(includes_pattern, o)]
return [o for o in includes if o not in excludes]
def match_local(self, prefix, includes, excludes):
"""
Filters os.walk() with include and exclude patterns.
See: http://stackoverflow.com/a/5141829/93559
"""
includes_pattern = r"|".join([fnmatch.translate(x) for x in includes])
excludes_pattern = r"|".join([fnmatch.translate(x) for x in excludes]) or r"$."
matches = []
for root, dirs, files in os.walk(prefix, topdown=True):
# exclude dirs
dirs[:] = [os.path.join(root, d) for d in dirs]
dirs[:] = [d for d in dirs if not re.match(excludes_pattern,
d.split(root)[1])]
# exclude/include files
files = [os.path.join(root, f) for f in files]
files = [os.path.join(root, f) for f in files
if not re.match(excludes_pattern, f)]
files = [os.path.join(root, f) for f in files
if re.match(includes_pattern, f.split(prefix)[1])]
for fname in files:
matches.append(fname)
return matches
def upload_files(self, abspaths, relpaths, remote_objects):
"""
Determines files to be uploaded and call ``upload_file`` on each.
"""
for relpath in relpaths:
abspath = [p for p in abspaths if p.endswith(relpath)][0]
cloud_datetime = remote_objects[relpath] if relpath in remote_objects else None
local_datetime = datetime.datetime.utcfromtimestamp(os.stat(abspath).st_mtime)
if cloud_datetime and local_datetime < cloud_datetime:
self.skip_count += 1
if not self.quiet:
print("Skipped {0}: not modified.".format(relpath))
continue
if relpath in remote_objects:
self.update_count += 1
else:
self.create_count += 1
self.upload_file(abspath, relpath)
def upload_file(self, abspath, cloud_filename):
"""
Uploads a file to the container.
"""
if not self.test_run:
content = open(abspath, "rb")
content_type = get_content_type(cloud_filename, content)
headers = get_headers(cloud_filename, content_type)
if headers.get("Content-Encoding") == "gzip":
content = get_gzipped_contents(content)
size = content.size
else:
size = os.stat(abspath).st_size
self.container.create(
obj_name=cloud_filename,
data=content,
content_type=content_type,
content_length=size,
content_encoding=headers.get("Content-Encoding", None),
headers=headers,
ttl=CUMULUS["FILE_TTL"],
etag=None,
)
self.upload_count += 1
if not self.quiet or self.verbosity > 1:
print("Uploaded: {0}".format(cloud_filename))
def delete_extra_files(self, relpaths, cloud_objs):
"""
Deletes any objects from the container that do not exist locally.
"""
for cloud_obj in cloud_objs:
if cloud_obj not in relpaths:
if not self.test_run:
self.delete_cloud_obj(cloud_obj)
self.delete_count += 1
if not self.quiet or self.verbosity > 1:
print("Deleted: {0}".format(cloud_obj))
def delete_cloud_obj(self, cloud_obj):
"""
Deletes an object from the container.
"""
self._connection.delete_object(
container=self.container_name,
obj=cloud_obj,
)
def wipe_container(self):
"""
Completely wipes out the contents of the container.
"""
if self.test_run:
print("Wipe would delete {0} objects.".format(len(self.container.object_count)))
else:
if not self.quiet or self.verbosity > 1:
print("Deleting {0} objects...".format(len(self.container.object_count)))
self._connection.delete_all_objects()
def print_tally(self):
"""
Prints the final tally to stdout.
"""
self.update_count = self.upload_count - self.create_count
if self.test_run:
print("Test run complete with the following results:")
print("Skipped {0}. Created {1}. Updated {2}. Deleted {3}.".format(
self.skip_count, self.create_count, self.update_count, self.delete_count))
class Command(NoArgsCommand):
help = "Synchronizes project static *or* media files to cloud files."
option_list = NoArgsCommand.option_list + (
optparse.make_option("-i", "--include", action="append", default=[],
dest="includes", metavar="PATTERN",
help="Include file or directories matching this glob-style "
"pattern. Use multiple times to include more."),
optparse.make_option("-e", "--exclude", action="append", default=[],
dest="excludes", metavar="PATTERN",
help="Exclude files or directories matching this glob-style "
"pattern. Use multiple times to exclude more."),
optparse.make_option("-w", "--wipe",
action="store_true", dest="wipe", default=False,
help="Wipes out entire contents of container first."),
optparse.make_option("-t", "--test-run",
action="store_true", dest="test_run", default=False,
help="Performs a test run of the sync."),
optparse.make_option("-q", "--quiet",
action="store_true", dest="test_run", default=False,
help="Do not display any output."),
optparse.make_option("-c", "--container",
dest="container", help="Override STATIC_CONTAINER."),
optparse.make_option("-s", "--static",
action="store_true", dest="syncstatic", default=False,
help="Sync static files located at settings.STATIC_ROOT path."),
optparse.make_option("-m", "--media",
action="store_true", dest="syncmedia", default=False,
help="Sync media files located at settings.MEDIA_ROOT path."),
)
def set_options(self, options):
"""
Sets instance variables based on an options dict
"""
# COMMAND LINE OPTIONS
self.wipe = options.get("wipe")
self.test_run = options.get("test_run")
self.quiet = options.get("test_run")
self.container_name = options.get("container")
self.verbosity = int(options.get("verbosity"))
self.syncmedia = options.get("syncmedia")
self.syncstatic = options.get("syncstatic")
if self.test_run:
self.verbosity = 2
cli_includes = options.get("includes")
cli_excludes = options.get("excludes")
# CUMULUS CONNECTION AND SETTINGS FROM SETTINGS.PY
if self.syncmedia and self.syncstatic:
raise CommandError("options --media and --static are mutually exclusive")
if not self.container_name:
if self.syncmedia:
self.container_name = CUMULUS["CONTAINER"]
elif self.syncstatic:
self.container_name = CUMULUS["STATIC_CONTAINER"]
else:
raise CommandError("must select one of the required options, either --media or --static")
settings_includes = CUMULUS["INCLUDE_LIST"]
settings_excludes = CUMULUS["EXCLUDE_LIST"]
# PATH SETTINGS
if self.syncmedia:
self.file_root = os.path.abspath(settings.MEDIA_ROOT)
self.file_url = settings.MEDIA_URL
elif self.syncstatic:
self.file_root = os.path.abspath(settings.STATIC_ROOT)
self.file_url = settings.STATIC_URL
if not self.file_root.endswith("/"):
self.file_root = self.file_root + "/"
if self.file_url.startswith("/"):
self.file_url = self.file_url[1:]
# SYNCSTATIC VARS
# combine includes and excludes from the cli and django settings file
self.includes = list(set(cli_includes + settings_includes))
self.excludes = list(set(cli_excludes + settings_excludes))
# transform glob patterns to regular expressions
self.local_filenames = []
self.create_count = 0
self.upload_count = 0
self.update_count = 0
self.skip_count = 0
self.delete_count = 0
def handle_noargs(self, *args, **options):
# setup
self.set_options(options)
self._connection = Auth()._get_connection()
self.container = self._connection.get_container(self.container_name)
# wipe first
if self.wipe:
self.wipe_container()
# match local files
abspaths = self.match_local(self.file_root, self.includes, self.excludes)
relpaths = []
for path in abspaths:
filename = path.split(self.file_root)[1]
if filename.startswith("/"):
filename = filename[1:]
relpaths.append(filename)
if not relpaths:
settings_root_prefix = "MEDIA" if self.syncmedia else "STATIC"
raise CommandError("The {0}_ROOT directory is empty "
"or all files have been ignored.".format(settings_root_prefix))
for path in abspaths:
if not os.path.isfile(path):
raise CommandError("Unsupported filetype: {0}.".format(path))
# match cloud objects
cloud_objs = self.match_cloud(self.includes, self.excludes)
remote_objects = {
obj.name: datetime.datetime.strptime(
obj.last_modified,
"%Y-%m-%dT%H:%M:%S.%f") for obj in self.container.get_objects()
}
# sync
self.upload_files(abspaths, relpaths, remote_objects)
self.delete_extra_files(relpaths, cloud_objs)
if not self.quiet or self.verbosity > 1:
self.print_tally()
def match_cloud(self, includes, excludes):
"""
Returns the cloud objects that match the include and exclude patterns.
"""
cloud_objs = [cloud_obj.name for cloud_obj in self.container.get_objects()]
includes_pattern = r"|".join([fnmatch.translate(x) for x in includes])
excludes_pattern = r"|".join([fnmatch.translate(x) for x in excludes]) or r"$."
excludes = [o for o in cloud_objs if re.match(excludes_pattern, o)]
includes = [o for o in cloud_objs if re.match(includes_pattern, o)]
return [o for o in includes if o not in excludes]
def match_local(self, prefix, includes, excludes):
"""
Filters os.walk() with include and exclude patterns.
See: http://stackoverflow.com/a/5141829/93559
"""
includes_pattern = r"|".join([fnmatch.translate(x) for x in includes])
excludes_pattern = r"|".join([fnmatch.translate(x) for x in excludes]) or r"$."
matches = []
for root, dirs, files in os.walk(prefix, topdown=True):
# exclude dirs
dirs[:] = [os.path.join(root, d) for d in dirs]
dirs[:] = [d for d in dirs if not re.match(excludes_pattern,
d.split(root)[1])]
# exclude/include files
files = [os.path.join(root, f) for f in files]
files = [os.path.join(root, f) for f in files
if not re.match(excludes_pattern, f)]
files = [os.path.join(root, f) for f in files
if re.match(includes_pattern, f.split(prefix)[1])]
for fname in files:
matches.append(fname)
return matches
def upload_files(self, abspaths, relpaths, remote_objects):
"""
Determines files to be uploaded and call ``upload_file`` on each.
"""
for relpath in relpaths:
abspath = [p for p in abspaths if p[len(self.file_root):] == relpath][0]
cloud_datetime = remote_objects[relpath] if relpath in remote_objects else None
local_datetime = datetime.datetime.utcfromtimestamp(os.stat(abspath).st_mtime)
if cloud_datetime and local_datetime < cloud_datetime:
self.skip_count += 1
if not self.quiet:
print("Skipped {0}: not modified.".format(relpath))
continue
if relpath in remote_objects:
self.update_count += 1
else:
self.create_count += 1
self.upload_file(abspath, relpath)
def upload_file(self, abspath, cloud_filename):
"""
Uploads a file to the container.
"""
if not self.test_run:
content = open(abspath, "rb")
content_type = get_content_type(cloud_filename, content)
headers = get_headers(cloud_filename, content_type)
if headers.get("Content-Encoding") == "gzip":
content = get_gzipped_contents(content)
size = content.size
else:
size = os.stat(abspath).st_size
self.container.create(
obj_name=cloud_filename,
data=content,
content_type=content_type,
content_length=size,
content_encoding=headers.get("Content-Encoding", None),
headers=headers,
ttl=CUMULUS["FILE_TTL"],
etag=None,
)
self.upload_count += 1
if not self.quiet or self.verbosity > 1:
print("Uploaded: {0}".format(cloud_filename))
def delete_extra_files(self, relpaths, cloud_objs):
"""
Deletes any objects from the container that do not exist locally.
"""
for cloud_obj in cloud_objs:
if cloud_obj not in relpaths:
if not self.test_run:
self.delete_cloud_obj(cloud_obj)
self.delete_count += 1
if not self.quiet or self.verbosity > 1:
print("Deleted: {0}".format(cloud_obj))
def delete_cloud_obj(self, cloud_obj):
"""
Deletes an object from the container.
"""
self._connection.delete_object(
container=self.container_name,
obj=cloud_obj,
)
def wipe_container(self):
"""
Completely wipes out the contents of the container.
"""
if self.test_run:
print("Wipe would delete {0} objects.".format(len(self.container.object_count)))
else:
if not self.quiet or self.verbosity > 1:
print("Deleting {0} objects...".format(len(self.container.object_count)))
self._connection.delete_all_objects()
def print_tally(self):
"""
Prints the final tally to stdout.
"""
self.update_count = self.upload_count - self.create_count
if self.test_run:
print("Test run complete with the following results:")
print("Skipped {0}. Created {1}. Updated {2}. Deleted {3}.".format(
self.skip_count, self.create_count, self.update_count, self.delete_count))
class Command(CollectStaticCommand):
"""
Command that allows to copy or symlink media files from Rackspace Cloud
Files location to the settings.MEDIA_ROOT.
"""
help = "Collect media files in a single location."
def __init__(self, *args, **kwargs):
super(Command, self).__init__(*args, **kwargs)
self.copied_files = []
self.symlinked_files = []
self.unmodified_files = []
self.post_processed_files = []
self.storage = swiftclient_storage
self.style = no_style()
try:
self.storage.path('')
except NotImplementedError:
self.local = False
else:
self.local = True
def set_options(self, **options):
"""
Set instance variables based on an options dict
"""
self.interactive = options['interactive']
self.verbosity = options['verbosity']
self.symlink = options['link']
self.clear = options['clear']
self.dry_run = options['dry_run']
ignore_patterns = options['ignore_patterns']
if options['use_default_ignore_patterns']:
ignore_patterns += ['CVS', '.*', '*~']
self.ignore_patterns = list(set(ignore_patterns))
self.post_process = options['post_process']
self.container_name = CUMULUS["CONTAINER"]
self._connection = Auth()._get_connection()
self.container = self._connection.get_container(self.container_name)
def collect(self):
"""
Perform the bulk of the work of collectstatic.
Split off from handle() to facilitate testing.
"""
if self.symlink and not self.local:
raise CommandError("Can't symlink to a remote destination.")
if self.clear:
self.clear_dir('')
if self.symlink:
handler = self.link_file
else:
handler = self.copy_file
cloud_objects = self.container.get_objects()
for obj in cloud_objects:
if self.dry_run:
self.log("Pretending to copy '%s'" % obj.name, level=1)
else:
self.log("Copying '%s'" % obj.name, level=1)
obj.download(settings.MEDIA_ROOT)
# TODO: don't overwrite files with matching timestamps
# TODO: delete local files that don't match any remote objects
return {
'modified': [obj.name for obj in cloud_objects],
'unmodified': [],
'post_processed': [],
}
