def main(argv):
minecraft_dir = FilePath(argv[1])
assert minecraft_dir.isdir(), minecraft_dir
servers_filename = minecraft_dir.child(SERVERS_DAT)
servers = get_servers(servers_filename)
for server in servers:
status = server.server.status()
show_server_players(server, status)
return 0
def mapCallings(client, data_dir='data', template_root='templates'):
data_fp = FilePath(data_dir)
output_root = data_fp.child('output')
if not output_root.exists():
output_root.makedirs()
jenv = Environment(loader=FileSystemLoader(template_root))
jenv.filters['abbr'] = abbreviateCalling
jenv.globals['math'] = math
template = jenv.get_template('callingmap.html')
#members = client.getRawValue('member_list')
callings = client.getRawValue('members_with_callings')
no_calling = client.getRawValue('members_without_callings')
no_calling = [x for x in no_calling if x['age'] >= 12]
# get the groups and subgroups organized into dicts
groups = OrderedDict()
by_suborg = {}
for line in prefOrder:
if line.startswith('#'):
# heading
groups[line[1:].strip()] = OrderedDict()
elif line.strip():
subgroup_key = line.strip()
groups[groups.keys()
[-1]][subgroup_key] = by_suborg[subgroup_key] = []
# put each calling into the right subgroup
# also count the number of callings per person
calling_counts = defaultdict(lambda: 0)
for calling in callings:
suborg = calling['subOrgType'] or calling['organization']
by_suborg[suborg].append(calling)
calling_counts[calling['id']] += 1
fp = output_root.child('callingmap.html')
fp.setContent(
template.render(orgs=groups,
calling_counts=calling_counts,
no_calling=no_calling).encode('utf-8'))
print 'wrote', fp.path
def mapCallings(client, data_dir='data', template_root='templates'):
data_fp = FilePath(data_dir)
output_root = data_fp.child('output')
if not output_root.exists():
output_root.makedirs()
jenv = Environment(loader=FileSystemLoader(template_root))
jenv.filters['abbr'] = abbreviateCalling
jenv.globals['math'] = math
template = jenv.get_template('callingmap.html')
#members = client.getRawValue('member_list')
callings = client.getRawValue('members_with_callings')
no_calling = client.getRawValue('members_without_callings')
no_calling = [x for x in no_calling if x['age'] >= 12]
# get the groups and subgroups organized into dicts
groups = OrderedDict()
by_suborg = {}
for line in prefOrder:
if line.startswith('#'):
# heading
groups[line[1:].strip()] = OrderedDict()
elif line.strip():
subgroup_key = line.strip()
groups[groups.keys()[-1]][subgroup_key] = by_suborg[subgroup_key] = []
# put each calling into the right subgroup
# also count the number of callings per person
calling_counts = defaultdict(lambda:0)
for calling in callings:
suborg = calling['subOrgType'] or calling['organization']
by_suborg[suborg].append(calling)
calling_counts[calling['id']] += 1
fp = output_root.child('callingmap.html')
fp.setContent(template.render(
orgs=groups,
calling_counts=calling_counts,
no_calling=no_calling).encode('utf-8'))
print 'wrote', fp.path
class LDSClient(object):
vals = ValueGetters()
def __init__(self, root, username, password):
self._session = None
self.username = username
self.password = password
self.root = FilePath(root)
self.raw_root = self.root.child('raw')
if not self.root.exists():
self.root.makedirs()
if not self.raw_root.exists():
self.raw_root.makedirs()
self.photo_root = self.root.child('photos')
if not self.photo_root.exists():
self.photo_root.makedirs()
def assertOk(self, response):
if not response.ok:
sys.stderr.write('not okay: %r\n' % (response, ))
sys.stderr.write(repr(response.text)[:200] + '\n')
raise Exception('response not okay', response)
def log(self, message):
sys.stderr.write(message + '\n')
def authenticate(self):
if self._session:
return self._session
self.log('Signing in...')
s = self._session = requests.session()
r = s.get('https://ident.lds.org/sso/UI/Login')
r = s.post('https://ident.lds.org/sso/UI/Login',
params={
'IDToken1': self.username,
'IDToken2': self.password,
'IDButton': 'Log In',
})
self.assertOk(r)
return self._session
def storeRawValue(self, filename, value):
self.raw_root.child(filename).setContent(json.dumps(value))
def hasRawValue(self, filename):
fp = self.raw_root.child(filename)
if fp.exists():
return fp
return None
def getRawValue(self, filename, default_value=None):
fp = self.hasRawValue(filename)
if fp:
return json.loads(fp.getContent())
else:
return default_value
def updateRawData(self):
for name, func in self.vals.raws.items():
if self.hasRawValue(name):
# already has a value; do not recompute
self.log('[%s] data already present' % (name, ))
continue
self.log('[%s] fetching...' % (name, ))
data = func(self)
self.storeRawValue(name, data)
@vals.raw('unit_number')
def get_unitNumber(self):
s = self.authenticate()
r = s.get('https://www.lds.org/mls/mbr/records/member-list?lang=eng')
self.assertOk(r)
# this is probably pretty fragile...
re_unit_number = re.compile(r"window.unitNumber\s=\s'(.*?)';")
m = re_unit_number.search(r.text)
return m.groups()[0]
@vals.raw('member_list')
def get_memberList(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/report/member-list',
params={
'lang': 'eng',
'unitNumber': unit_number,
})
self.assertOk(r)
return r.json()
@vals.raw('members_with_callings')
def get_membersWithCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get(
'https://www.lds.org/mls/mbr/services/report/members-with-callings',
params={
'lang': 'eng',
'unitNumber': unit_number,
},
headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
@vals.raw('members_without_callings')
def get_membersWithoutCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get(
'https://www.lds.org/mls/mbr/services/orgs/members-without-callings',
params={
'lang': 'eng',
'unitNumber': unit_number,
},
headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
#----------------------------
# photos
#----------------------------
def _memberPhotoFilePath(self, member_id, size='large', ext='jpg'):
"""
Valid size options are:
- large
- medium
- original
- thumbnail
"""
return self.photo_root.child('solo-%s-%s.%s' % (member_id, size, ext))
def _memberIDsWithNoPhoto(self, size='large'):
members = self.getRawValue('member_list')
for member in members:
member_id = member['id']
photo_fp = self._memberPhotoFilePath(member_id, size)
if photo_fp.exists():
continue
yield member_id
def updatePhotos(self, size='large'):
s = self.authenticate()
self.log('Getting photos...')
for member_ids in xAtATime(self._memberIDsWithNoPhoto(size), 19):
if not member_ids:
continue
try:
r = s.get(
'https://www.lds.org/directory/services/ludrs/photo/url/' +
','.join(map(str, member_ids)) + '/individual')
data = r.json()
except ValueError:
print 'Error on', member_ids
raise
for member_id, result in zip(member_ids, data):
fp = self._memberPhotoFilePath(member_id, size)
uri = result[size + 'Uri']
if uri:
print 'fetching photo for', member_id
uri = 'https://www.lds.org' + uri
image_data = s.get(uri)
content_type = image_data.headers['content-type']
if content_type != 'image/jpeg':
print 'NON-JPEG: ', content_type, member_id
continue
fp.setContent(image_data.content)
else:
print 'no photo for', member_id
time.sleep(0.5)
USE_TZ = True
# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''
# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''
# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = PROJECT_ROOT.child("static").path
# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'
# Additional locations of static files
STATICFILES_DIRS = (
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
PROJECT_ROOT.preauthChild("code_scouts_platform/static").path,
)
# List of finder classes that know how to find static files in
# various locations.
from __future__ import unicode_literals
from yaml import safe_load
from testtools import skipUnless
from testtools.matchers import AfterPreprocessing, Equals
from lae_util.testtools import TestCase
from filepath import FilePath
from lae_automation import __file__ as lae_package_location
ROOT = FilePath(lae_package_location).parent().parent()
STAGING = ROOT.child("k8s").child("secrets.staging.enc.yaml")
PRODUCTION = ROOT.child("k8s").child("secrets.production.enc.yaml")
class SecretsFileTests(TestCase):
"""
Tests for the files in the repository which contain the S4 secrets.
"""
@skipUnless(
STAGING.exists() and PRODUCTION.exists(),
"Staging and production secrets missing.",
)
def test_same_staging_and_production_structure(self):
"""
The staging and production secrets files have the same overall structure
(the same keys and hierarchy).
class LDSClient(object):
vals = ValueGetters()
def __init__(self, root, username, password):
self._session = None
self.username = username
self.password = password
self.root = FilePath(root)
self.raw_root = self.root.child('raw')
if not self.root.exists():
self.root.makedirs()
if not self.raw_root.exists():
self.raw_root.makedirs()
self.photo_root = self.root.child('photos')
if not self.photo_root.exists():
self.photo_root.makedirs()
def assertOk(self, response):
if not response.ok:
sys.stderr.write('not okay: %r\n' % (response,))
sys.stderr.write(repr(response.text)[:200] + '\n')
raise Exception('response not okay', response)
def log(self, message):
sys.stderr.write(message + '\n')
def authenticate(self):
if self._session:
return self._session
self.log('Signing in...')
s = self._session = requests.session()
r = s.get('https://ident.lds.org/sso/UI/Login')
r = s.post('https://ident.lds.org/sso/UI/Login', params={
'IDToken1': self.username,
'IDToken2': self.password,
'IDButton': 'Log In',
})
self.assertOk(r)
return self._session
def storeRawValue(self, filename, value):
self.raw_root.child(filename).setContent(json.dumps(value))
def hasRawValue(self, filename):
fp = self.raw_root.child(filename)
if fp.exists():
return fp
return None
def getRawValue(self, filename, default_value=None):
fp = self.hasRawValue(filename)
if fp:
return json.loads(fp.getContent())
else:
return default_value
def updateRawData(self):
for name, func in self.vals.raws.items():
if self.hasRawValue(name):
# already has a value; do not recompute
self.log('[%s] data already present' % (name,))
continue
self.log('[%s] fetching...' % (name,))
data = func(self)
self.storeRawValue(name, data)
@vals.raw('unit_number')
def get_unitNumber(self):
s = self.authenticate()
r = s.get('https://www.lds.org/mls/mbr/records/member-list?lang=eng')
self.assertOk(r)
# this is probably pretty fragile...
re_unit_number = re.compile(r"window.unitNumber\s=\s'(.*?)';")
m = re_unit_number.search(r.text)
return m.groups()[0]
@vals.raw('member_list')
def get_memberList(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/report/member-list', params={
'lang': 'eng',
'unitNumber': unit_number,
})
self.assertOk(r)
return r.json()
@vals.raw('members_with_callings')
def get_membersWithCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/report/members-with-callings', params={
'lang': 'eng',
'unitNumber': unit_number,
}, headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
@vals.raw('members_without_callings')
def get_membersWithoutCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/orgs/members-without-callings', params={
'lang': 'eng',
'unitNumber': unit_number,
}, headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
#----------------------------
# photos
#----------------------------
def _memberPhotoFilePath(self, member_id, size='large', ext='jpg'):
"""
Valid size options are:
- large
- medium
- original
- thumbnail
"""
return self.photo_root.child('solo-%s-%s.%s' % (member_id, size, ext))
def _memberIDsWithNoPhoto(self, size='large'):
members = self.getRawValue('member_list')
for member in members:
member_id = member['id']
photo_fp = self._memberPhotoFilePath(member_id, size)
if photo_fp.exists():
continue
yield member_id
def updatePhotos(self, size='large'):
s = self.authenticate()
self.log('Getting photos...')
for member_ids in xAtATime(self._memberIDsWithNoPhoto(size), 19):
if not member_ids:
continue
try:
r = s.get('https://www.lds.org/directory/services/ludrs/photo/url/'+','.join(map(str, member_ids))+'/individual')
data = r.json()
except ValueError:
print 'Error on', member_ids
raise
for member_id, result in zip(member_ids, data):
fp = self._memberPhotoFilePath(member_id, size)
uri = result[size + 'Uri']
if uri:
print 'fetching photo for', member_id
uri = 'https://www.lds.org' + uri
image_data = s.get(uri)
content_type = image_data.headers['content-type']
if content_type != 'image/jpeg':
print 'NON-JPEG: ', content_type, member_id
continue
fp.setContent(image_data.content)
else:
print 'no photo for', member_id
time.sleep(0.5)
