def main(argv): args = argv[1:] # Defaults key_id = get_s3_info('key_id') secret = get_s3_info('key') size = DEFAULT_SIZE bucket = None prefix = None while args: a = args.pop(0) if a == '-C' and args: filename = args.pop(0) args = file(filename).read().split() + args elif a == '-a' and args: key_id = args.pop(0) elif a == '-s' and args: secret = args.pop(0) elif a == '-l' and args: size = parse_size(args.pop(0)) elif a == '-mcproperties': import s3options bucket = s3options.get_bucket() key_id = s3options.get_key_id() secret = s3options.get_secret() elif a.startswith('-'): usage() elif bucket is None: bucket = a elif prefix is None: prefix = a else: usage() if not prefix or not bucket: usage() if not key_id: print >>stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id" exit(1) if not secret: print >>stderr, "Must either specify -s {secret} or put secret into ~/.s3/key" exit(1) s3.init0(key_id = key_id, secret = secret) segment = 0 while True: block = read_fully(stdin, size) if not block: break object = prefix + '.%03d' % segment s3.put_data(bucket, object, block) print object, len(block) segment += 1
def main(argv): global debug args = argv[1:] username = DEFAULT_USERNAME pid_file = PID_FILE log_file = LOG_FILE try: s3.init0() except: pass while args: if args[0] == '-D': debug = True args = args[1:] elif args[0] == '-C' and len(args) > 1: args = file(args[1]).read().split() + args[2:] elif args[0] == '-u' and len(args) > 1: username = args[1] args = args[2:] elif args[0] == '-pid' and len(args) > 1: pid_file = args[1] args = args[2:] elif args[0] == '-log' and len(args) > 1: log_file = args[1] args = args[2:] elif args[0] == '-db' and len(args) > 1: db_options['db'] = args[1] args = args[2:] elif args[0] == '-user' and len(args) > 1: db_options['user'] = args[1] args = args[2:] elif args[0] == '-host' and len(args) > 1: cxp_options['host'] = args[1] args = args[2:] elif args[0] == '-bucket' and len(args) > 1: cxp_options['bucket'] = args[1] args = args[2:] elif args[0] == '-a' and len(args) > 1: s3.KEY_ID = args[1] args = args[2:] elif args[0] == '-s' and len(args) > 1: s3.SECRET = args[1] args = args[2:] elif args[0] == '-kfile' and len(args) > 1: cxp_options['kfile'] = expanduser(args[1]) args = args[2:] else: usage() if not s3.KEY_ID or not s3.SECRET: usage() if not debug: try: from os import _exit, setuid, setgid, fork, dup2, open, close, \ mkdir, chdir, O_RDONLY, O_WRONLY, O_APPEND, O_CREAT from pwd import getpwnam except ImportError: pass else: pw = getpwnam(username) pid = fork() if pid: print >> file(pid_file, 'w'), pid _exit(0) fd = open('/dev/null', O_RDONLY) dup2(fd, 0) fd = open(log_file, O_WRONLY | O_APPEND | O_CREAT) dup2(fd, 1) dup2(fd, 2) close(fd) setgid(pw.pw_gid) setuid(pw.pw_uid) while True: db = sql.connect(**db_options) c = db.cursor() c.execute("SELECT id, account_id, guid, amazon_user_token, amazon_product_token" + \ " FROM backup_queue, mcx.users" + \ " WHERE endtime IS NULL AND account_id = mcx.users.mcid" + \ " ORDER BY id" + \ " LIMIT 1") t = c.fetchone() if t: c.execute("UPDATE backup_queue" + \ " SET starttime=NOW(), status='UPLOADING'" + \ " WHERE id=%s;" % t[0]) db.commit() c.close() db.close() if t: put(t[0], t[1], t[2], t[3], t[4]) else: sleep(1)
def main(argv): global debug args = argv[1:] username = DEFAULT_USERNAME pid_file = PID_FILE log_file = LOG_FILE try: s3.init0() except: pass while args: if args[0] == '-D': debug = True args = args[1:] elif args[0] == '-C' and len(args) > 1: args = file(args[1]).read().split() + args[2:] elif args[0] == '-u' and len(args) > 1: username = args[1] args = args[2:] elif args[0] == '-pid' and len(args) > 1: pid_file = args[1] args = args[2:] elif args[0] == '-log' and len(args) > 1: log_file = args[1] args = args[2:] elif args[0] == '-db' and len(args) > 1: db_options['db'] = args[1] args = args[2:] elif args[0] == '-user' and len(args) > 1: db_options['user'] = args[1] args = args[2:] elif args[0] == '-host' and len(args) > 1: cxp_options['host'] = args[1] args = args[2:] elif args[0] == '-bucket' and len(args) > 1: cxp_options['bucket'] = args[1] args = args[2:] elif args[0] == '-a' and len(args) > 1: s3.KEY_ID = args[1] args = args[2:] elif args[0] == '-s' and len(args) > 1: s3.SECRET = args[1] args = args[2:] elif args[0] == '-kfile' and len(args) > 1: cxp_options['kfile'] = expanduser(args[1]) args = args[2:] else: usage() if not s3.KEY_ID or not s3.SECRET: usage() if not debug: try: from os import _exit, setuid, setgid, fork, dup2, open, close, \ mkdir, chdir, O_RDONLY, O_WRONLY, O_APPEND, O_CREAT from pwd import getpwnam except ImportError: pass else: pw = getpwnam(username) pid = fork() if pid: print >>file(pid_file, 'w'), pid _exit(0) fd = open('/dev/null', O_RDONLY) dup2(fd, 0) fd = open(log_file, O_WRONLY | O_APPEND | O_CREAT) dup2(fd, 1) dup2(fd, 2) close(fd) setgid(pw.pw_gid) setuid(pw.pw_uid) while True: db = sql.connect(**db_options) c = db.cursor() c.execute("SELECT id, account_id, guid, amazon_user_token, amazon_product_token" + \ " FROM backup_queue, mcx.users" + \ " WHERE endtime IS NULL AND account_id = mcx.users.mcid" + \ " ORDER BY id" + \ " LIMIT 1") t = c.fetchone() if t: c.execute("UPDATE backup_queue" + \ " SET starttime=NOW(), status='UPLOADING'" + \ " WHERE id=%s;" % t[0]) db.commit() c.close() db.close() if t: put(t[0], t[1], t[2], t[3], t[4]) else: sleep(1)
def main(argv): args = argv[1:] # Defaults key_id = get_s3_info('key_id') secret = get_s3_info('key') bucket = None object = None output = dup(1) while args: a = args.pop(0) if a == '-C' and args: filename = args.pop(0) args = file(filename).read().split() + args elif a == '-a' and args: key_id = args.pop(0) elif a == '-s' and args: secret = args.pop(0) elif a == '-mcproperties': import s3options bucket = s3options.get_bucket() key_id = s3options.get_key_id() secret = s3options.get_secret() elif a == '-o': close(output) output = open(args.pop(0), O_WRONLY | O_CREAT | O_TRUNC, 0600) elif a.startswith('-'): usage() elif bucket is None: bucket = a elif object is None: object = a else: usage() if not bucket or not object: usage() if not key_id: print >>stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id" exit(1) if not secret: print >>stderr, "Must either specify -s {secret} or put secret into ~/.s3/key" exit(1) s3.init0(key_id = key_id, secret = secret) if object.count('%%') == object.count('%') * 2: # No strftime formatting instructions download(bucket, object.replace('%%', '%'), output) else: for target_date in date_range(date.today()): if download(bucket, target_date.strftime(object), output): break close(output)
def main(argv): args = argv[1:] # Defaults key_id = get_s3_info('key_id') secret = get_s3_info('key') bucket = None host = None while args: a = args.pop(0) if a == '-C' and args: filename = args.pop(0) args = file(filename).read().split() + args elif a == '-a' and args: key_id = args.pop(0) elif a == '-s' and args: secret = args.pop(0) elif a == '-mcproperties': import s3options bucket = s3options.get_bucket() key_id = s3options.get_key_id() secret = s3options.get_secret() elif a.startswith('-'): usage() elif bucket is None: bucket = a elif host is None: host = a else: usage() if not host: host = getfqdn() if not bucket: usage() if not key_id: print >> stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id" exit(1) if not secret: print >> stderr, "Must either specify -s {secret} or put secret into ~/.s3/key" exit(1) s3.init0(key_id=key_id, secret=secret) today = date.today() if today.weekday() == SUNDAY: target = today - 5 * WEEK else: target = today - WEEK if target.day != 1: # NOT the first of the month, safe to delete prefix = '%s-%s' % (host, target.strftime('%Y-%m-%d')) delete_all_segments(bucket, prefix + '.sql.bz2.enc') delete_all_segments(bucket, prefix + '.tar.bz2.enc')
def main(argv): args = argv[1:] # Defaults key_id = get_s3_info('key_id') secret = get_s3_info('key') size = DEFAULT_SIZE bucket = None prefix = None while args: a = args.pop(0) if a == '-C' and args: filename = args.pop(0) args = file(filename).read().split() + args elif a == '-a' and args: key_id = args.pop(0) elif a == '-s' and args: secret = args.pop(0) elif a == '-l' and args: size = parse_size(args.pop(0)) elif a == '-mcproperties': import s3options bucket = s3options.get_bucket() key_id = s3options.get_key_id() secret = s3options.get_secret() elif a.startswith('-'): usage() elif bucket is None: bucket = a elif prefix is None: prefix = a else: usage() if not prefix or not bucket: usage() if not key_id: print >> stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id" exit(1) if not secret: print >> stderr, "Must either specify -s {secret} or put secret into ~/.s3/key" exit(1) s3.init0(key_id=key_id, secret=secret) segment = 0 while True: block = read_fully(stdin, size) if not block: break object = prefix + '.%03d' % segment s3.put_data(bucket, object, block) print object, len(block) segment += 1
def main(argv): args = argv[1:] # Defaults key_id = get_s3_info('key_id') secret = get_s3_info('key') bucket = None host = None while args: a = args.pop(0) if a == '-C' and args: filename = args.pop(0) args = file(filename).read().split() + args elif a == '-a' and args: key_id = args.pop(0) elif a == '-s' and args: secret = args.pop(0) elif a == '-mcproperties': import s3options bucket = s3options.get_bucket() key_id = s3options.get_key_id() secret = s3options.get_secret() elif a.startswith('-'): usage() elif bucket is None: bucket = a elif host is None: host = a else: usage() if not host: host = getfqdn() if not bucket: usage() if not key_id: print >>stderr, "Must either specify -a {key_id} or put key_id into ~/.s3/key_id" exit(1) if not secret: print >>stderr, "Must either specify -s {secret} or put secret into ~/.s3/key" exit(1) s3.init0(key_id = key_id, secret = secret) today = date.today() if today.weekday() == SUNDAY: target = today - 5 * WEEK else: target = today - WEEK if target.day != 1: # NOT the first of the month, safe to delete prefix = '%s-%s' % (host, target.strftime('%Y-%m-%d')) delete_all_segments(bucket, prefix + '.sql.bz2.enc') delete_all_segments(bucket, prefix + '.tar.bz2.enc')