def InitAndRun(run_callback,
shutdown_callback=None,
scan_ops=False,
server_logging=True):
"""Runs the main initialization routine, with the provided run and shutdown callbacks wrapped
by the WWW-specific functionality provided by '_StartWWW' and '_ShutdownWWW'.
"""
main.InitAndRun(run_callback=partial(_StartWWW, run_callback, scan_ops),
shutdown_callback=shutdown_callback,
server_logging=server_logging)
if got_lock == False:
logging.warning('Failed to acquire job lock: exiting.')
callback()
return
result = None
job.Start()
try:
result = yield gen.Task(RunOnce, client, job)
except:
# Failure: log run summary with trace.
typ, val, tb = sys.exc_info()
msg = ''.join(traceback.format_exception(typ, val, tb))
logging.info('Registering failed run with message: %s' % msg)
yield gen.Task(job.RegisterRun, Job.STATUS_FAILURE, failure_msg=msg)
else:
if result is not None and not options.options.dry_run:
# Successful run with data processed and not in dry-run mode: write run summary.
stats = DotDict()
stats['last_day'] = result
logging.info('Registering successful run with stats: %r' % stats)
yield gen.Task(job.RegisterRun, Job.STATUS_SUCCESS, stats=stats)
finally:
yield gen.Task(job.ReleaseLock)
callback()
if __name__ == '__main__':
sys.exit(main.InitAndRun(_Start))
for user in users:
last_user_id = user.user_id
if options.min_user_id != -1 and user.user_id < options.min_user_id:
continue
if options.max_user_id != -1 and user.user_id > options.max_user_id:
continue
# Only send to users which allow marketing communication to them.
if options.honor_allow_marketing:
settings = yield gen.Task(AccountSettings.QueryByUser, client,
user.user_id, None)
if not settings.AllowMarketing():
continue
if user.email:
yield SendEmailToUser(email_template, user)
elif user.phone:
if not options.sms_template:
if not sms_warning:
print 'WARNING: no SMS template specified and phone-only accounts encountered; skipping...'
sms_warning = True
else:
yield SendSMSToUser(sms_template, user)
callback()
if __name__ == '__main__':
sys.exit(main.InitAndRun(Run))
"""Upgrades each table in 'options.options.tables'."""
if options.options.verbose_test_run:
logging.info(
'***** NOTE: upgrade is being run in verbose testing mode; run with '
'--verbose_test_run=False once changes have been verified')
Version.SetMutateItems(False)
if options.options.allow_s3_queries is not None:
logging.info('Setting allow_s3_queries=%r' %
options.options.allow_s3_queries)
Version.SetAllowS3Queries(options.options.allow_s3_queries)
OpManager.SetInstance(
OpManager(op_map=DB_OPERATION_MAP,
client=db_client.DBClient.Instance()))
tables = [vf_schema.SCHEMA.GetTable(n) for n in options.options.tables]
if not tables:
raise Exception(
'The --tables option has not been specified. ' +
'Tables to upgrade must be explicitly listed using this option.')
yield [
gen.Task(UpgradeTable, db_client.DBClient.Instance(), table)
for table in tables
]
callback()
if __name__ == '__main__':
sys.exit(main.InitAndRun(Upgrade))
if a.startswith('-'):
new_args.append(a)
else:
commands.append(a)
options.parse_command_line(args=new_args)
if len(commands) == 0:
PrintHelp()
sys.exit(0)
cmd = commands[0]
cmd_args = commands[1:]
if cmd == 'bench':
cb = Benchmark
elif cmd == 'cat':
cb = Cat
elif cmd == 'grep':
cb = Grep
elif cmd == 'ls':
cb = List
elif cmd == 'mv':
cb = Move
elif cmd == 'put':
cb = Put
else:
print 'Unknown command: %s' % cmd
PrintHelp()
sys.exit(0)
sys.exit(main.InitAndRun(partial(cb, cmd_args), server_logging=False))
def RunDBA(callback):
"""Runs op on each table listed in --tables."""
logging.warning('WARNING: this tool can modify low-level DynamoDB tables and '
'attributes and should be used with caution. For example, '
'modifying a photo or adding a label directly will '
'not update secondary indexes nor create user updates.')
def _OnInit(verified_schema):
if options.options.op == 'list':
def _OnList(result):
logging.info(result)
callback()
db_client.DBClient.Instance().ListTables(callback=_OnList)
else:
if options.options.tables == 'ALL':
tables = vf_schema.SCHEMA.GetTables()
else:
tables = [vf_schema.SCHEMA.GetTable(n) for n in options.options.tables]
assert tables, 'no tables were specified'
with util.Barrier(callback) as b:
for table in tables:
RunOpOnTable(db_client.DBClient.Instance(), table, options.options.op, b.Callback())
db_client.InitDB(vf_schema.SCHEMA, callback=_OnInit,
verify_or_create=options.options.verify_or_create)
if __name__ == '__main__':
sys.exit(main.InitAndRun(RunDBA, init_db=False))
if key.range_key:
attrs[table.range_key_col.name] = key.range_key
logging.info('attributes: %s' % attrs)
def _OnOp(*args, **kwargs):
if args:
logging.info('positional result args: %s' % pprint.pformat(args))
if kwargs:
logging.info('keyword result args: %s' % pprint.pformat(kwargs))
callback()
if op in ('query', 'update'):
assert table, 'no table name specified for operation'
if op in ('query'):
assert table.range_key_col, 'Table %s is not composite' % table.name
# Run the operation
logging.info('executing %s' % op)
if op == 'query':
QueryRange(client, table, cls, key, range_desc, _OnOp)
elif op == 'update':
o = cls()
o.UpdateFromKeywords(**attrs)
o.Update(client, _OnOp)
else:
raise Exception('unrecognized op: %s' % op)
if __name__ == '__main__':
sys.exit(main.InitAndRun(RunDBA))
from viewfinder.backend.db import db_client, vf_schema
from viewfinder.backend.db.admin_permissions import AdminPermissions
options.define('user', default=None, help='user to set/delete')
options.define('rights', default=None, multiple=True, help='list of rights ("root", "support"). Omit to clear.')
options.define('op', default=None, help='command: set, del')
def ProcessAdmins(callback):
client = db_client.DBClient.Instance()
assert options.options.user is not None
assert options.options.op is not None
if options.options.op == 'set':
permissions = AdminPermissions(options.options.user, options.options.rights)
logging.info('committing %r' % permissions)
permissions.Update(client, callback)
elif options.options.op == 'del':
admin = AdminPermissions(options.options.user)
logging.info('deleting %r' % admin)
admin.Delete(client, callback)
else:
logging.error('unknown op: %s' % options.options.op)
callback()
if __name__ == '__main__':
sys.exit(main.InitAndRun(ProcessAdmins))
Usage: python -m viewfinder.backend.storage.url_generator --key=key """ __author__ = '[email protected] (Spencer Kimball)' import logging import sys import time from tornado import httpclient, ioloop, options from viewfinder.backend.base import main, secrets from viewfinder.backend.storage.s3_object_store import S3ObjectStore options.define('key', default='', help='S3 asset key') def GenerateURL(callback): """Runs op on each table listed in --tables.""" object_store = S3ObjectStore('photos-viewfinder-co') upload_url = object_store.GenerateUploadUrl(options.options.key) logging.info('PUT URL: %s' % upload_url) logging.info('GET URL: %s' % object_store.GenerateUrl(options.options.key)) callback() if __name__ == '__main__': sys.exit(main.InitAndRun(GenerateURL))
