def main(argv):
args = argv[1:]
# Defaults
key_id = get_s3_info('key_id')
secret = get_s3_info('key')
size = DEFAULT_SIZE
bucket = None
prefix = None
while args:
a = args.pop(0)
if a == '-C' and args:
filename = args.pop(0)
args = file(filename).read().split() + args
elif a == '-a' and args:
key_id = args.pop(0)
elif a == '-s' and args:
secret = args.pop(0)
elif a == '-l' and args:
size = parse_size(args.pop(0))
elif a == '-mcproperties':
import s3options
bucket = s3options.get_bucket()
key_id = s3options.get_key_id()
secret = s3options.get_secret()
elif a.startswith('-'):
usage()
elif bucket is None:
bucket = a
elif prefix is None:
prefix = a
else:
usage()
if not prefix or not bucket:
usage()
if not key_id:
print >>stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id"
exit(1)
if not secret:
print >>stderr, "Must either specify -s {secret} or put secret into ~/.s3/key"
exit(1)
s3.init0(key_id = key_id, secret = secret)
segment = 0
while True:
block = read_fully(stdin, size)
if not block:
break
object = prefix + '.%03d' % segment
s3.put_data(bucket, object, block)
print object, len(block)
segment += 1
def main(argv):
global debug
args = argv[1:]
username = DEFAULT_USERNAME
pid_file = PID_FILE
log_file = LOG_FILE
try:
s3.init0()
except:
pass
while args:
if args[0] == '-D':
debug = True
args = args[1:]
elif args[0] == '-C' and len(args) > 1:
args = file(args[1]).read().split() + args[2:]
elif args[0] == '-u' and len(args) > 1:
username = args[1]
args = args[2:]
elif args[0] == '-pid' and len(args) > 1:
pid_file = args[1]
args = args[2:]
elif args[0] == '-log' and len(args) > 1:
log_file = args[1]
args = args[2:]
elif args[0] == '-db' and len(args) > 1:
db_options['db'] = args[1]
args = args[2:]
elif args[0] == '-user' and len(args) > 1:
db_options['user'] = args[1]
args = args[2:]
elif args[0] == '-host' and len(args) > 1:
cxp_options['host'] = args[1]
args = args[2:]
elif args[0] == '-bucket' and len(args) > 1:
cxp_options['bucket'] = args[1]
args = args[2:]
elif args[0] == '-a' and len(args) > 1:
s3.KEY_ID = args[1]
args = args[2:]
elif args[0] == '-s' and len(args) > 1:
s3.SECRET = args[1]
args = args[2:]
elif args[0] == '-kfile' and len(args) > 1:
cxp_options['kfile'] = expanduser(args[1])
args = args[2:]
else:
usage()
if not s3.KEY_ID or not s3.SECRET:
usage()
if not debug:
try:
from os import _exit, setuid, setgid, fork, dup2, open, close, \
mkdir, chdir, O_RDONLY, O_WRONLY, O_APPEND, O_CREAT
from pwd import getpwnam
except ImportError:
pass
else:
pw = getpwnam(username)
pid = fork()
if pid:
print >> file(pid_file, 'w'), pid
_exit(0)
fd = open('/dev/null', O_RDONLY)
dup2(fd, 0)
fd = open(log_file, O_WRONLY | O_APPEND | O_CREAT)
dup2(fd, 1)
dup2(fd, 2)
close(fd)
setgid(pw.pw_gid)
setuid(pw.pw_uid)
while True:
db = sql.connect(**db_options)
c = db.cursor()
c.execute("SELECT id, account_id, guid, amazon_user_token, amazon_product_token" + \
" FROM backup_queue, mcx.users" + \
" WHERE endtime IS NULL AND account_id = mcx.users.mcid" + \
" ORDER BY id" + \
" LIMIT 1")
t = c.fetchone()
if t:
c.execute("UPDATE backup_queue" + \
" SET starttime=NOW(), status='UPLOADING'" + \
" WHERE id=%s;" % t[0])
db.commit()
c.close()
db.close()
if t:
put(t[0], t[1], t[2], t[3], t[4])
else:
sleep(1)
def main(argv):
global debug
args = argv[1:]
username = DEFAULT_USERNAME
pid_file = PID_FILE
log_file = LOG_FILE
try:
s3.init0()
except:
pass
while args:
if args[0] == '-D':
debug = True
args = args[1:]
elif args[0] == '-C' and len(args) > 1:
args = file(args[1]).read().split() + args[2:]
elif args[0] == '-u' and len(args) > 1:
username = args[1]
args = args[2:]
elif args[0] == '-pid' and len(args) > 1:
pid_file = args[1]
args = args[2:]
elif args[0] == '-log' and len(args) > 1:
log_file = args[1]
args = args[2:]
elif args[0] == '-db' and len(args) > 1:
db_options['db'] = args[1]
args = args[2:]
elif args[0] == '-user' and len(args) > 1:
db_options['user'] = args[1]
args = args[2:]
elif args[0] == '-host' and len(args) > 1:
cxp_options['host'] = args[1]
args = args[2:]
elif args[0] == '-bucket' and len(args) > 1:
cxp_options['bucket'] = args[1]
args = args[2:]
elif args[0] == '-a' and len(args) > 1:
s3.KEY_ID = args[1]
args = args[2:]
elif args[0] == '-s' and len(args) > 1:
s3.SECRET = args[1]
args = args[2:]
elif args[0] == '-kfile' and len(args) > 1:
cxp_options['kfile'] = expanduser(args[1])
args = args[2:]
else:
usage()
if not s3.KEY_ID or not s3.SECRET:
usage()
if not debug:
try:
from os import _exit, setuid, setgid, fork, dup2, open, close, \
mkdir, chdir, O_RDONLY, O_WRONLY, O_APPEND, O_CREAT
from pwd import getpwnam
except ImportError:
pass
else:
pw = getpwnam(username)
pid = fork()
if pid:
print >>file(pid_file, 'w'), pid
_exit(0)
fd = open('/dev/null', O_RDONLY)
dup2(fd, 0)
fd = open(log_file, O_WRONLY | O_APPEND | O_CREAT)
dup2(fd, 1)
dup2(fd, 2)
close(fd)
setgid(pw.pw_gid)
setuid(pw.pw_uid)
while True:
db = sql.connect(**db_options)
c = db.cursor()
c.execute("SELECT id, account_id, guid, amazon_user_token, amazon_product_token" + \
" FROM backup_queue, mcx.users" + \
" WHERE endtime IS NULL AND account_id = mcx.users.mcid" + \
" ORDER BY id" + \
" LIMIT 1")
t = c.fetchone()
if t:
c.execute("UPDATE backup_queue" + \
" SET starttime=NOW(), status='UPLOADING'" + \
" WHERE id=%s;" % t[0])
db.commit()
c.close()
db.close()
if t:
put(t[0], t[1], t[2], t[3], t[4])
else:
sleep(1)
def main(argv):
args = argv[1:]
# Defaults
key_id = get_s3_info('key_id')
secret = get_s3_info('key')
bucket = None
object = None
output = dup(1)
while args:
a = args.pop(0)
if a == '-C' and args:
filename = args.pop(0)
args = file(filename).read().split() + args
elif a == '-a' and args:
key_id = args.pop(0)
elif a == '-s' and args:
secret = args.pop(0)
elif a == '-mcproperties':
import s3options
bucket = s3options.get_bucket()
key_id = s3options.get_key_id()
secret = s3options.get_secret()
elif a == '-o':
close(output)
output = open(args.pop(0), O_WRONLY | O_CREAT | O_TRUNC, 0600)
elif a.startswith('-'):
usage()
elif bucket is None:
bucket = a
elif object is None:
object = a
else:
usage()
if not bucket or not object:
usage()
if not key_id:
print >>stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id"
exit(1)
if not secret:
print >>stderr, "Must either specify -s {secret} or put secret into ~/.s3/key"
exit(1)
s3.init0(key_id = key_id, secret = secret)
if object.count('%%') == object.count('%') * 2:
# No strftime formatting instructions
download(bucket, object.replace('%%', '%'), output)
else:
for target_date in date_range(date.today()):
if download(bucket, target_date.strftime(object), output):
break
close(output)
def main(argv):
args = argv[1:]
# Defaults
key_id = get_s3_info('key_id')
secret = get_s3_info('key')
bucket = None
host = None
while args:
a = args.pop(0)
if a == '-C' and args:
filename = args.pop(0)
args = file(filename).read().split() + args
elif a == '-a' and args:
key_id = args.pop(0)
elif a == '-s' and args:
secret = args.pop(0)
elif a == '-mcproperties':
import s3options
bucket = s3options.get_bucket()
key_id = s3options.get_key_id()
secret = s3options.get_secret()
elif a.startswith('-'):
usage()
elif bucket is None:
bucket = a
elif host is None:
host = a
else:
usage()
if not host:
host = getfqdn()
if not bucket:
usage()
if not key_id:
print >> stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id"
exit(1)
if not secret:
print >> stderr, "Must either specify -s {secret} or put secret into ~/.s3/key"
exit(1)
s3.init0(key_id=key_id, secret=secret)
today = date.today()
if today.weekday() == SUNDAY:
target = today - 5 * WEEK
else:
target = today - WEEK
if target.day != 1:
# NOT the first of the month, safe to delete
prefix = '%s-%s' % (host, target.strftime('%Y-%m-%d'))
delete_all_segments(bucket, prefix + '.sql.bz2.enc')
delete_all_segments(bucket, prefix + '.tar.bz2.enc')
def main(argv):
args = argv[1:]
# Defaults
key_id = get_s3_info('key_id')
secret = get_s3_info('key')
size = DEFAULT_SIZE
bucket = None
prefix = None
while args:
a = args.pop(0)
if a == '-C' and args:
filename = args.pop(0)
args = file(filename).read().split() + args
elif a == '-a' and args:
key_id = args.pop(0)
elif a == '-s' and args:
secret = args.pop(0)
elif a == '-l' and args:
size = parse_size(args.pop(0))
elif a == '-mcproperties':
import s3options
bucket = s3options.get_bucket()
key_id = s3options.get_key_id()
secret = s3options.get_secret()
elif a.startswith('-'):
usage()
elif bucket is None:
bucket = a
elif prefix is None:
prefix = a
else:
usage()
if not prefix or not bucket:
usage()
if not key_id:
print >> stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id"
exit(1)
if not secret:
print >> stderr, "Must either specify -s {secret} or put secret into ~/.s3/key"
exit(1)
s3.init0(key_id=key_id, secret=secret)
segment = 0
while True:
block = read_fully(stdin, size)
if not block:
break
object = prefix + '.%03d' % segment
s3.put_data(bucket, object, block)
print object, len(block)
segment += 1
def main(argv):
args = argv[1:]
# Defaults
key_id = get_s3_info('key_id')
secret = get_s3_info('key')
bucket = None
host = None
while args:
a = args.pop(0)
if a == '-C' and args:
filename = args.pop(0)
args = file(filename).read().split() + args
elif a == '-a' and args:
key_id = args.pop(0)
elif a == '-s' and args:
secret = args.pop(0)
elif a == '-mcproperties':
import s3options
bucket = s3options.get_bucket()
key_id = s3options.get_key_id()
secret = s3options.get_secret()
elif a.startswith('-'):
usage()
elif bucket is None:
bucket = a
elif host is None:
host = a
else:
usage()
if not host:
host = getfqdn()
if not bucket:
usage()
if not key_id:
print >>stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id"
exit(1)
if not secret:
print >>stderr, "Must either specify -s {secret} or put secret into ~/.s3/key"
exit(1)
s3.init0(key_id = key_id, secret = secret)
today = date.today()
if today.weekday() == SUNDAY:
target = today - 5 * WEEK
else:
target = today - WEEK
if target.day != 1:
# NOT the first of the month, safe to delete
prefix = '%s-%s' % (host, target.strftime('%Y-%m-%d'))
delete_all_segments(bucket, prefix + '.sql.bz2.enc')
delete_all_segments(bucket, prefix + '.tar.bz2.enc')
