def handle(self, *args, **options):
app_id = os.environ.get("APPLICATION_ID")
host = "%s.appspot.com" % app_id
remote_api_stub.ConfigureRemoteDatastore(app_id, '/remote_api',
auth_func, host)
code.interact('App Engine interactive console for %s' % (app_id, ),
None, locals())
def remote(args, context=None):
"""Starts a shell with the datastore as remote_api_stub.
Args:
args: arguments from the user
context: locals that should be added to the shell
"""
from google.appengine.api import apiproxy_stub_map
from google.appengine.ext import db
from google.appengine.ext.remote_api import remote_api_stub
if not context:
context = {}
app_id = args[0]
if len(args) > 1:
host = args[1]
else:
host = '%s.appspot.com' % app_id
remote_api_stub.ConfigureRemoteDatastore(app_id, '/remote_api', auth_func, host)
context['deepFetch'] = deepFetch
code.interact('App Engine interactive console for %s' % (app_id,), None, context)
def main(): helpString = " Usage: python %s your_restore_file.xml <optional new url-safe rakontu short-name to copy data into>" % (sys.argv[0]) if len(sys.argv) > 1 and sys.argv[1] in ["--help", "-help", "help", "-H", "-h"]: print helpString sys.exit() else: if len(sys.argv) < 2: print helpString else: if len(sys.argv) > 2: newRakontuShortName = sys.argv[2] else: newRakontuShortName = None restoreFileName = findRestoreFile(RESTORE_DIR, sys.argv[1]) prompt = "Restore from %s to %s" % (sys.argv[1], SERVER_NAME) if newRakontuShortName: prompt += ", creating a new Rakontu named %s" % newRakontuShortName prompt += "? (Y/n) " confirm = raw_input(prompt) if confirm == "Y": if os.path.exists(restoreFileName): try: remote_api_stub.ConfigureRemoteDatastore(APP_ID, '/remote_api', auth_func, servername=SERVER_NAME) restore(restoreFileName, newRakontuShortName) except KeyboardInterrupt: print "\nOK - quitting." sys.exit() else: print "Could not find file: %s" % restoreFileName else: print "OK - not restoring. Come again :)"
def setup_env(app_id, host=None, auth_func=None):
"""Setup remote access to a GAE instance."""
auth_func = auth_func or smart_auth_func
host = host or '%s.appspot.com' % app_id
# pylint: disable=W0612
from google.appengine.api import memcache
from google.appengine.api.users import User
from google.appengine.ext import db
from google.appengine.ext import ndb
remote_api_stub.ConfigureRemoteDatastore(
None, '/_ah/remote_api', auth_func, host)
# Initialize environment.
os.environ['SERVER_SOFTWARE'] = ''
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
import appengine_config
# Create shortcuts.
import codereview
from codereview import models, models_chromium, views, views_chromium
# Symbols presented to the user.
predefined_vars = locals().copy()
del predefined_vars['appengine_config']
del predefined_vars['auth_func']
# Load all the models.
for i in dir(models):
if re.match(r'[A-Z][a-z]', i[:2]):
predefined_vars[i] = getattr(models, i)
for i in dir(models_chromium):
if re.match(r'[A-Z][a-z]', i[:2]):
predefined_vars[i] = getattr(models_chromium, i)
return predefined_vars
def main():
for line in file('/etc/resolv.conf'):
if line.startswith('nameserver'):
nameserver, ip = line.strip().split()
if ip not in NAMESERVERS:
NAMESERVERS.append(ip)
from optparse import OptionParser
parser = OptionParser()
parser.add_option('-v', '--verbose', action='store_true',
help="output all names, not only existing")
parser.add_option('--server', metavar='<hostname>',
default='scoretool.appspot.com',
help="connect to a different server")
parser.add_option('--batch', metavar='<size>', type="int", default=100,
help="adjust batch size (default 100)")
parser.add_option('--timeout', metavar='<seconds>', type="int", default=20,
help="maximum wait time for DNS response (default 20)")
parser.add_option('--min', metavar='<length>', type="int", default=3,
help="only names of this length or longer (default 3)")
parser.add_option('--max', metavar='<length>', type="int", default=9,
help="only names of this length or shorter (default 9)")
parser.add_option('--left', metavar='<keyword>', default=None,
help="only names with this prefix")
parser.add_option('--right', metavar='<keyword>', default=None,
help="only names with this suffix")
parser.add_option('--random', action='store_true',
help="update random popular prefixes and suffixes")
parser.add_option('--retry', metavar='<error>',
help="update e.g. timeout=20 or status=rcodeservfail")
parser.add_option('--resume', metavar='<name>',
help="continue file upload from this name")
parser.add_option('--days', metavar='<number>', type='int',
help="update lookups younger than <number> days")
(options, args) = parser.parse_args()
remote_api_stub.ConfigureRemoteDatastore(
'scoretool', '/remote_api_hidden', auth_func, options.server)
if args:
upload_files(args, options)
elif options.random:
while True:
update_random(options)
elif options.retry:
options.active_tld_list = TOP_LEVEL_DOMAINS[:]
while options.active_tld_list:
update_error(options)
elif options.left is not None:
for length in range(max(options.min, len(options.left)),
options.max + 1):
update_best_names('left', options.left, length, options)
elif options.right is not None:
for length in range(max(options.min, len(options.right)),
options.max + 1):
update_best_names('right', options.right, length, options)
else:
while True:
update_oldest_lookups(options)
def run_from_argv(self, argv):
app_id = argv[2]
if len(argv) > 3:
host = argv[3]
else:
host = '%s.appspot.com' % app_id
remote_api_stub.ConfigureRemoteDatastore(app_id, '/remote_api',
auth_func, host)
code.interact('App Engine interactive console for %s' % (app_id, ),
None, locals())
def main():
if len(sys.argv) < 2:
print "Usage: %s app_id [host]" % (sys.argv[0],)
app_id = sys.argv[1]
if len(sys.argv) > 2:
host = sys.argv[2]
else:
host = '%s.appspot.com' % app_id
remote_api_stub.ConfigureRemoteDatastore(app_id, '/remote_api',
auth_func, host)
code.interact('App Engine interactive console for %s' % (app_id,),
None, locals())
def main():
if len(sys.argv) < 2:
app_id = 'chromium-status'
else:
app_id = sys.argv[1]
if len(sys.argv) > 2:
host = sys.argv[2]
else:
host = '%s.appspot.com' % app_id
logging.basicConfig(level=logging.ERROR)
# pylint: disable=W0612
from google.appengine.api import memcache
from google.appengine.ext import db
remote_api_stub.ConfigureRemoteDatastore(app_id, '/_ah/remote_api',
auth_func, host)
from appengine_module.chromium_status import base_page
from appengine_module.chromium_status import breakpad
from appengine_module.chromium_status import event_push
from appengine_module.chromium_status import lkgr
from appengine_module.chromium_status import profiling
from appengine_module.chromium_status import static_blobs_inline
from appengine_module.chromium_status import static_blobs_store
from appengine_module.chromium_status import status
from appengine_module.chromium_status import utils
from appengine_module.chromium_status import xmpp
utils.bootstrap()
def remove(entity, functor, batch=100):
"""Remove entries."""
count = 0
items = []
while True:
entries = [
i for i in entity.all().fetch(limit=batch) if functor(i)
]
count += len(entries)
print '%s' % count
if entries:
db.delete(entries)
else:
break
# Symbols presented to the user.
predefined_vars = locals()
prompt = ('App Engine interactive console for "%s".\n'
'Available symbols:\n'
' %s\n') % (app_id, ', '.join(sorted(predefined_vars)))
code.interact(prompt, None, predefined_vars)
def authorize(self, options):
"""
Setup all the GAE remote API bizness.
"""
# Pull the app id
app_id = self.get_app_id()
# Figure out the URL to hit
if options.get('host'):
host = options.get('host')
else:
host = '%s.appspot.com' % app_id
# Connect
remote_api_stub.ConfigureRemoteDatastore(app_id, '/remote_api', self.login, host)
def setupRemote(app_id, host=None):
"""Sets up execution for the specified remote.
"""
from google.appengine.api import apiproxy_stub_map
from google.appengine.ext import db
from google.appengine.ext.remote_api import remote_api_stub
if not host:
host = '%s.appspot.com' % app_id.split("~")[-1]
remote_api_stub.ConfigureRemoteDatastore(app_id, '/_ah/remote_api',
auth_func, host)
remote_api_stub.MaybeInvokeAuthentication()
def setup_remote(self):
self.remote = True
logging.info('Setting up remote_api for "%s" at http://%s%s' %
(self.remote_app_id, self.remote_host, self.remote_url))
from google.appengine.ext.remote_api import remote_api_stub
from google.appengine.ext import db
remote_api_stub.ConfigureRemoteDatastore(
self.remote_app_id,
self.remote_url,
auth_func,
self.remote_host,
rpc_server_factory=rpc_server_factory)
logging.info('Now using the remote datastore for "%s" at http://%s%s' %
(self.remote_app_id, self.remote_host, self.remote_url))
def main():
from optparse import OptionParser
parser = OptionParser()
parser.add_option('--server',
metavar='<hostname>',
default='scoretool.appspot.com',
help="connect to a different server")
(options, args) = parser.parse_args()
remote_api_stub.ConfigureRemoteDatastore('scoretool', '/remote_api',
auth_func, options.server)
while True:
whois_keys = Whois.all(keys_only=True).fetch(BATCH_SIZE)
if not whois_keys:
break
delete_batch(whois_keys)
def connect(server, app_id=None, username=None, password=None):
"""Sets up a connection to an app that has the remote_api handler."""
if not app_id:
app_id = get_app_id()
print 'Application ID: %s' % app_id
print 'Server: %s' % server
if not username:
username = raw_input('Username: '******'Username: %s' % username
# Sets up users.get_current_user() inside of the console
os.environ['USER_EMAIL'] = username
if not password:
password = getpass.getpass('Password: '******'/remote_api', lambda: (username, password), server)
db.Query().count() # force authentication to happen now
def main(): if len(sys.argv) > 1: if sys.argv[1] in ["--help", "-help", "help", "-H", "-h"]: print " Usage: python %s <optional rakontu short-name to backup, otherwise all>" % (sys.argv[0]) sys.exit() else: rakontuShortNameToBackup = sys.argv[1] prompt = "Backup Rakontu %s from %s to %s? (Y/n) " % (rakontuShortNameToBackup, SERVER_NAME, BACKUP_DIR) else: rakontuShortNameToBackup = None prompt = "Backup all Rakontus from %s to %s? (Y/n) " % (SERVER_NAME, BACKUP_DIR) confirm = raw_input(prompt) if confirm == "Y": if os.path.exists(BACKUP_DIR): try: remote_api_stub.ConfigureRemoteDatastore(APP_ID, '/remote_api', auth_func, servername=SERVER_NAME) backup(rakontuShortNameToBackup) except KeyboardInterrupt: print "\nOK - quitting." sys.exit() else: print "Directory does not exist: %s" % BACKUP_DIR else: print "OK - not backing up. Come again :)"
def connect(host=None):
if host is None:
host = conf.CHIRPRADIO_HOST
remote_api_stub.ConfigureRemoteDatastore(
"s~chirpradio-hrd", "/_ah/remote_api", _auth_func, host)
#!/usr/bin/python
import code
import getpass
import sys
sys.path.append("/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine")
sys.path.append("/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/yaml/lib")
from google.appengine.ext.remote_api import remote_api_stub
from google.appengine.ext import db
def auth_func():
return raw_input('Username:'******'Password:'******'%s.appspot.com' % app_id
remote_api_stub.ConfigureRemoteDatastore(app_id, '/remote_api', auth_func, host)
code.interact('App Engine interactive console for %s' % (app_id,), None, locals())
######################################################################
# Main
# Configure connection via remote_api to datastore - after this
# data store calls are remote
log("Connecting to " + options.host)
def auth_func():
return (options.email, getpass.getpass('Password:'******'theyworkforyouelection',
'/remote_api',
auth_func,
servername=options.host)
# Load in extra files
if options.fetch:
log("Fetching latest YourNextMP JSON file")
ynmp_url = YOURNEXTMP_URL % (settings.YOURNEXTMP_API_TOKEN)
ynmp_file = "/tmp/load_freshdata_yournextmp.json"
ynmp_h = open(ynmp_file + ".gz", 'w')
ynmp_h.write(urllib2.urlopen(ynmp_url).read())
ynmp_h.close()
ynmp_h = open(ynmp_file, 'w')
ynmp_h.write(gzip.GzipFile(ynmp_file + ".gz").read())
ynmp_h.close()
args.append(ynmp_file)
'timeout=30',
]
def auth_func():
if os.path.exists(PASSWORD_FILENAME):
return open(PASSWORD_FILENAME).read().split(':')
username = raw_input('Username:'******'Password:'******'scoretool', '/remote_api_hidden',
auth_func, 'scoretool.appspot.com')
while PURGE_VALUES:
query = Lookup.all(keys_only=True)
value = random.choice(PURGE_VALUES)
query.filter('com', value)
keys = retry(query.fetch, BATCH_SIZE)
if len(keys) < BATCH_SIZE:
PURGE_VALUES.remove(value)
if len(keys):
print "deleting %d names (%s to %s) where com is %s" % (
len(keys), keys[0].name(), keys[-1].name(), value)
domain_keys = [
db.Key.from_path('domains_domain', key.name()) for key in keys
]
db.delete(keys + domain_keys)
logging.warn("Caught URLError exception, retrying.")
continue
if skip_this_batch:
pass
elif in_batches:
yield(entities)
else:
for entity in entities:
yield(entity)
if len(entities) < batch_size and not skip_this_batch:
break
q.with_cursor(q.cursor())
def each_batch(*args, **kwargs):
'''A generator that runs across all elements of a data store model in batches.
'''
return each(*args, in_batches=True, **kwargs)
if username is None:
username = raw_input('Username: '******'Password: '******'/_ah/remote_api', auth_func, host)
from mp import handlers as mps
os.environ["USER_EMAIL"] = "" # Otherwise the dev appserver can have assertion failures
code.interact('App Engine interactive console for %s on %s' % (APP_ID, host), None, locals())
import dev_appserver
dev_appserver.fix_sys_path()
from google.appengine.api.app_identity import get_application_id
from google.appengine.ext.remote_api import remote_api_stub
APP_NAME = get_application_id()
os.environ['APPLICATION_ID'] = APP_NAME
print "APPNAME", app_name
os.environ['AUTH_DOMAIN'] = GAE_USERNAME
os.environ['USER_EMAIL'] = GAE_USERNAME
servername = APP_NAME + ".appspot.com"
if USE_DEV:
APP_NAME = "dev~" + APP_NAME
servername = "localhost:8080"
else:
APP_NAME = "s~" + APP_NAME
password = gitpass.gitpass('GAE Password')
def auth_function():
return (GAE_USERNAME, password)
remote_api_stub.ConfigureRemoteDatastore(APP_NAME,
'/_ah/remote_api',
auth_function,
servername=servername)
from lib.models import *
APP_NAME = 'russiaflatrent'
os.environ['AUTH_DOMAIN'] = 'gmail.com'
os.environ['USER_EMAIL'] = '*****@*****.**'
def auth_func():
return ("*****@*****.**", "calestoffs4ic")
# return (raw_input('Username:'******'Password:'******'localhost:8080'
# Otherwise, remote_api assumes you are targeting APP_NAME.appspot.com
remote_api_stub.ConfigureRemoteDatastore(APP_NAME, '/remote_api', auth_func)
# Do stuff like your code was running on App Engine
from adsparser import *
url = "http://www.cian.ru/cat.php?deal_type=1¤cy=2&totime=3600&room0=1&room1=1&room2=1&room3=1&room4=1&obl_id=1&order=4"
import time
class CianAdParser(BaseAdParser):
def date_format(self):
return "%d.%m.%y %H:%M"
def encoding(self):
filename = '.data/popular/%ses.%d.txt' % (kind.split('_')[-1], length)
outfile = open(filename, 'w')
start = db.Key.from_path(kind, '-')
while True:
query = Model.all().filter('length', length).order('__key__')
query.filter('__key__ >', start)
prefixes = retry(query.fetch, 1000)
for prefix in prefixes:
name = prefix.key().name()
if name.startswith('.'):
continue
if kind == 'prefixes_suffix':
name = name[::-1]
outfile.write('%d %s\n' % (prefix.com, name))
if len(prefixes) < 1000:
break
start = prefixes[-1].key()
outfile.close()
def auth_func():
return open('.passwd').read().split(':')
if __name__ == '__main__':
remote_api_stub.ConfigureRemoteDatastore('scoretool', '/remote_api_hidden',
auth_func, 'www.nxdom.com')
for length in range(2, 7):
fetch_file(Prefix, length)
fetch_file(Suffix, length)
two = {}
total = 0
for c1 in DOMAIN_CHARS:
query = Prefix.all().filter('length', 2)
query.filter('__key__ >', db.Key.from_path('prefixes_prefix', c1))
next = chr(ord(c1) + 1)
query.filter('__key__ <', db.Key.from_path('prefixes_prefix', next))
one[c1] = 0
for prefix in query.fetch(100):
total += prefix.count
one[c1] += prefix.count
two[prefix.key().name()] = prefix.count
print 'TOTAL =', total
print
print_counts('ONE', one)
print
print_counts('TWO', two)
if __name__ == '__main__':
from optparse import OptionParser
parser = OptionParser()
parser.add_option('--server',
metavar='<hostname>',
default='scoretool.appspot.com',
help="connect to a different server")
(options, args) = parser.parse_args()
remote_api_stub.ConfigureRemoteDatastore('scoretool', '/remote_api',
auth_func, options.server)
main()
def login(self):
host = '%s.appspot.com' % self.appid
remote_api_stub.ConfigureRemoteDatastore(self.appid, '/remote_api',
auth_func, host)
return ('*****@*****.**', 'woodspy2008')
# return (raw_input('Username:'******'Password:'******'%Y-%m-%d %H:%M:%S',)
# return str(time.mktime(timestamp))[:10]
# Use local dev server by passing in as parameter:
# servername='localhost:8080'
# Otherwise, remote_api assumes you are targeting APP_NAME.appspot.com
remote_api_stub.ConfigureRemoteDatastore(
APP_NAME,
'/remote_api',
auth_func,
servername='ihere.appspot.com',
# servername='localhost:8080',
)
# Do stuff like your code was running on App Engine
#foos = Foo.all().fetch(100)
#for foo in foos:
# foo.note = 'Hello World!'
#db.puts(foos)
#timestamp=str(time.time())[:10]
#key_name='post_'+timestamp
#
#post=db.put(Post(
# key_name=key_name,
#!/usr/bin/python
import code
import getpass
import sys
import os
sys.path.append("/usr/local/google_appengine")
sys.path.append("/usr/local/google_appengine/lib/yaml/lib")
sys.path= [os.path.join(os.path.dirname(__file__), 'shared'), os.path.join(os.path.dirname(__file__), 'Apps')]+sys.path
from google.appengine.ext.remote_api import remote_api_stub
from google.appengine.ext import db
from google.appengine.dist import use_library
use_library('django', '1.1')
import os, cgi
os.environ['DJANGO_SETTINGS_MODULE']='techevents_settings'
def auth_func():
return 'rosskarchner', '1GingerBeer'
host = 'localhost:8083'
remote_api_stub.ConfigureRemoteDatastore('techevents', '/remote_api', auth_func, host)
code.interact('App Engine interactive console for %s' % ('techevents'), None, locals())
