def saveWorkload(name, content):
import tempfile
wkld_dir = os.path.join(
config.getSettings("global")["projdir"],
config.getSettings("workloads")["dir"]
)
ensureDir(wkld_dir)
temp_f = tempfile.mkstemp(
suffix='.wkld',
prefix='',
dir=wkld_dir
)
try:
temp = os.fdopen(temp_f[0], 'wb')
temp.write(content)
finally:
temp.close()
newWorkload = {
"name" : name,
"file" : temp_f[1],
"actors" : []
}
if name in config.getSettings("workloads")["defs"]:
os.remove(config.getSettings("workloads")["defs"][name]["file"])
config.getSettings("workloads")["defs"][name] = newWorkload
config.saveConfig()
def GET(self, request):
"""
Get a particular page.
The handler will take anything after the base address and look for it
in the pages folder. This folder is stored in the :mod:`config` module
at::
config.getSettings("server")["pagedir"]
If a file is not found the handler will assume the address is a
directory and try to serve the file index.html from it.
:returns: The requested file.
:raises: :exc:`restlite.Status` 400 if the path contains ".."
:exc:`restlite.Status` 404 if the file is not found.
"""
authentication.login(request)
if '..' in request['PATH_INFO']:
raise restlite.Status, '400 Invalid Path'
page_file = os.path.join(
config.getSettings("global")["basedir"],
config.getSettings("server")["pagedir"],
request['PATH_INFO']
)
try:
return request.response(self.getPage(page_file), self.getType(page_file))
except restlite.Status:
index_page = os.path.join(page_file, 'index.html')
return request.response(self.getPage(index_page), 'text/html')
def setupConfig(proj_dir, base_dir):
"""
Setup the :mod:`config` module using the config file that will be or is in
the project directory.
The project and base directories will only be stored if the config is new.
:param proj_dir: The absolute path to the project directory.
:type proj_dir: ``str``
:param base_dir: The absolute path to the base (server installation)
directory.
:type base_dir: ``str``
:returns: ``True`` if there was not an existing config file. (i.e. this was
a new configuration.)
:rtype: ``bool``
"""
config_file = os.path.join(proj_dir, pb.appname_file+".config")
print "Using " + config_file + " as the config file."
config.setConfig(config_file)
new_config = not config.onDisk()
if new_config:
print "Storing essential directories."
config.getSettings("global")["basedir"] = base_dir
config.getSettings("global")["projdir"] = proj_dir
else:
print "Config Loaded."
return new_config
def makeKey():
"""
Generate a new :program:`ssh` key to be used by vazels.
This will be very insecure as Vazels requires a passwordless key.
If you wish to inspect the generated product, it will be saved to
the path returned by :func:`keyPath()`.
"""
ensureSecurityDir()
try:
os.remove(keyPath())
os.remove(keyPath()+".pub")
except: pass
command = [
"ssh-keygen",
"-t", "rsa",
"-f", keyPath(),
"-q",
"-C", "Generated by Reef for use with Vazels",
"-N", "",
]
subprocess.Popen(
command,
stdout=open('/dev/null','w'),
stderr=subprocess.STDOUT
).wait()
config.getSettings("security")["customkey"] = True
config.getSettings("security")["certificate"] = keyPath()
def saveActor(name, type, content):
import tempfile
actor_dir = os.path.join(
config.getSettings("global")["projdir"],
config.getSettings("actors")["dir"]
)
ensureDir(actor_dir)
temp_f = tempfile.mkstemp(
suffix='.tar.gz',
prefix='',
dir=actor_dir
)
try:
temp = os.fdopen(temp_f[0], 'wb')
temp.write(content)
finally:
temp.close()
newActor = {
"name" : name,
"type" : type,
"file" : temp_f[1]
}
if name in config.getSettings("actors")["defs"]:
os.remove(config.getSettings("actors")["defs"][name]["file"])
config.getSettings("actors")["defs"][name] = newActor
config.saveConfig()
def _extractSueComponents(group_number, sueComps):
"""
Extract SUE components into the appropriate directories.
:param group_number: The number of the group to extract into.
:type group_number: ``int``
:param sueComps: The set of names of Sue Componenets to extract.
:type sueComps: ``set(str)``
"""
experiment_dir = config.getSettings('control_centre')['experiment_dir']
for sueComp_name in sueComps:
try:
sueCompFile = config.getSettings('SUE')['defs'][sueComp_name]['file']
except KeyError:
print sueComp_name
print config.getSettings('SUE')['defs']
raise restlite.Status, "500, Failed to extract Sue component"
extractpath = os.path.join(
controlcentre.getExperimentPath(),
"Group_"+str(group_number),
"SUE"
)
sueCompTGZ = tarfile.open(sueCompFile,'r:gz')
sueCompTGZ.extractall(path=extractpath)
def restore():
if config.getSettings("security")["backedup"]:
ensureSecurityDir()
try:
shutil.move(backupFile(), authKeysFile())
# Failure means there was not backup, so delete file
except IOError:
try:
os.remove(authKeysFile())
except:
pass
config.getSettings("security")["backedup"] = True
def getControlCentreOutPath():
"""
Grab the control centre output file.
:returns: Absolute path to the ouput file.
:rtype: ``str``
"""
return os.path.join(
config.getSettings("global")["projdir"],
config.getSettings("control_centre")["out_file"]
)
def backup():
if not config.getSettings("security")["backedup"]:
ensureSecurityDir()
try:
os.remove(backupFile())
except:
pass # Means it probably didn't exist
try:
shutil.copy2(authKeysFile(), backupFile())
# Ignore failure as it just means there was nothing to backup
except IOError:
pass
config.getSettings("security")["backedup"] = True
def getVazelsPath():
"""
Grab the path to the vazels installation directory.
:returns: Absolute path to the vazels install directory.
:rtype: ``str``
"""
return os.path.join(
config.getSettings("global")['basedir'],
config.getSettings("control_centre")["vazels_dir"],
)
def getExperimentPath():
"""
Grab the experiment path.
:returns: Absolute path to the vazels experiment for this project.
:rtype: ``str``
"""
return os.path.join(
config.getSettings("global")['projdir'],
config.getSettings("control_centre")["experiment_dir"]
)
def GET(request):
authentication.login(request)
if ".." in request["PATH_INFO"]:
raise restlite.Status, "400 Invalid Path"
page_file = os.path.join(
config.getSettings("global")["basedir"], config.getSettings("server")["pagedir"], request["PATH_INFO"]
)
try:
return request.response(getPage(page_file), getType(page_file))
except restlite.Status:
index_page = os.path.join(page_file, "index.html")
return request.response(getPage(index_page), "text/html")
def __extractActors(workload_def, target_group):
print("Extracting actors")
group_number = target_group['group_number']
experiment_dir = config.getSettings("global")['expdir']
print("workload_def: " + str(workload_def))
for actor_name in workload_def['actors']:
print("Extracting actor: " + actor_name)
actor = config.getSettings('actors')['defs'][actor_name]
type=actor['type'].upper()
actorTGZ = tarfile.open(actor['file'],'r:gz')
if type.find("SUE") != -1 :
actorTGZ.extractall(path=experiment_dir+'/Group_'+str(group_number)+'/SUE/')
else :
actorTGZ.extractall(path=experiment_dir+'/Group_'+str(group_number)+"/Vazels/"+type+"_launcher/")
def __getCommandLineClientArgs():
"""
Get the arguments needed to start the command line client.
:returns: List of the necessary arguments.
:rtype: ``list(str)``
"""
return [
'/bin/sh',
'commandline_client.sh',
'--rmi_host='+config.getSettings('control_centre')['rmi_host'],
'--rmi_port='+config.getSettings('control_centre')['rmi_port']
]
def __init__(self):
"""
Create a basic server setup.
This will set up the most basic possible settings for a working server.
It also adds enough defaults in the :mod:`config` so that the server
will run with no additional input.
"""
self.authmodel = None
self.routes = []
config.getSettings("server").setdefault("port", 8000)
# Paths from base dir
config.getSettings("server").setdefault("pagedir", "./pages/")
def updateFrequency():
cron = CronTab(user=True)
writeCron = False
settings = getSettings()
sidebarJob = getJob(cron, 'sidebar/main.py')
# If the job exists, make sure the existing crontab matches
# what the settings say it should be
if sidebarJob:
if sidebarJob.minute != '*/' + str(settings['sidebar']['update_interval_mins']):
sidebarJob.minute.every(settings['sidebar']['update_interval_mins'])
writeCron = True
# If the job doesn't exist, create it with the settings given
else:
# This crontab entry will run the bot and redirect stderr and stdout to
# an individual file for just this iteration as well as to the full
# appended output log that can be `tail -f`'d
base_path = os.path.dirname(os.path.abspath(sys.argv[0]))
sidebarJob = cron.new(command='python ' + base_path \
+ '/main.py 2>&1 | tee ' + base_path \
+ '/logs/iterations/`date -u +\%s`.txt >> ' \
+ base_path + '/logs/output.txt',
comment='GOB Sidebar module')
sidebarJob.minute.every(settings['sidebar']['update_interval_mins'])
writeCron = True
# If changes were made, write them to the crontab
if writeCron:
cron.write()
def POST(request,entity):
authentication.login(request)
existing_groups = config.getSettings("groups")
group_name = request['PATH_INFO']
args = urlparse.parse_qs(entity)
g_data = _getGroupDataFromArgs(args)
if group_name in existing_groups:
if "size" in g_data and g_data["size"] == 0:
# Delete the group
del existing_groups[group_name]
else:
existing_groups[group_name].update(g_data)
else:
# New group
n_group = __newGroup()
n_group["name"] = group_name
n_group.update(g_data)
if n_group["size"] != 0:
existing_groups[group_name] = n_group
#print config.getSettings
try:
return request.response(existing_groups[group_name])
except KeyError:
return request.response({"name":group_name, "size":0})
def loadCategories(self):
"""
Loads categories (task lists) from the net
Returns a dictionary containing the names (key) and IDs (value) for the categories.
"""
if not self.isConnected():
raise ValueError('Not connected with RTM Net Service; cannot proceed. Please connect first.')
categories = {}
rspLists = self._connection.lists.getList()
for cat in rspLists.lists.list:
try:
config.getSettings().getValue("hidden_groups", list).index(cat.name)
except:
categories[cat.name] = cat.id
return categories
def init_settings(self):
self.settings = config.getSettings()
self.settings.beginGroup('glclient')
self.port = unicode(self.settings.value('port').toString())
if not self.port:
self.port = '9999'
self.settings.setValue('port', self.port)
print 'new settings'
print unicode(self.settings.value('port').toString())
self.colormap = unicode(self.settings.value('colormap').toString())
if not self.colormap:
self.colormap = '../../gl/colormaps/rainbow2'
if not os.path.isfile(self.colormap):
print 'Colormap file not found:', self.colormap
self.colormap = unicode(QtGui.QFileDialog.getOpenFileName(self, 'Select Colormap File'))
self.settings.setValue('colormap', self.colormap)
self.executable = unicode(self.settings.value('executable').toString() )
if not self.executable:
self.executable = '../../gl/src/glclient'
if not os.access(self.executable, os.X_OK):
self.executable = QtGui.QFileDialog.getOpenFileName(self, 'Select GLClient Executable')
self.settings.setValue('executable', self.executable)
if not os.access(self.executable, os.X_OK):
QtGui.QMessageBox.critical(self, 'GLClient executable', 'Please select the correct glclient executable. Make sure you have execute permission on this file')
self.settings.endGroup()
def doWork():
"""
where we can put our work
:return:
"""
logging.info("looking for applicable alerts")
settings = config.getSettings()
saltStackMaster = settings["saltStackMaster"]
saltStackUser = settings["saltStackUser"]
saltStackPass = settings["saltStackPass"]
saltStackStateToRun = settings["saltStackStateToRun"]
alertsRaw = alerts.getAlertExists(True)
if alertsRaw["alertExists"]:
logging.info("Firing off Salt State since I see an alert")
session = requests.Session()
session.post('https://' + saltStackMaster + ':8000/login',
verify=False,
json={
'username': saltStackUser,
'password': saltStackUser,
'eauth': 'auto',
})
resp = session.post('https://' + saltStackMaster + ':8000/login',
verify=False,
json=[{
'client': 'local',
'tgt': '*',
'fun': 'state.orchestrate',
'arg': [saltStackStateToRun],
}])
logging.info(resp.content)
return resp
else:
return {"status": "nothing to do"}
def restore():
"""
Restore the backed up version of the :program:`ssh` authorized_hosts file if
a backup has been created by :func:`backup`.
"""
if config.getSettings("security")["backedup"]:
ensureSecurityDir()
try:
shutil.move(backupFile(), authKeysFile())
# Failure means there was not backup, so delete file
except IOError:
try: os.remove(authKeysFile())
except: pass
config.getSettings("security")["backedup"] = True
def GET(request):
authentication.login(request)
groups = config.getSettings("groups")
#component_info = {}
# For each component, build a list of groups to which it is assigned.
#for component_name in config.getSettings("SUE")["defs"]:
# assigned_groups = []
# for group in groups:
# if component_name in groups[group]["sue_components"]:
# assigned_groups.append(group)
# component_info[component_name] = assigned_groups
#return request.response(component_info)
return request.response(config.getSettings("SUE")["defs"].keys(), 'text/html')
def GET(self, request):
"""
Get setting data from the :mod:`config`.
Parameters are expected to be part of the query string.
If the base of this address handler is called:
:returns: A list of sections in the :mod:`config`
:rtype: ``list``
If anything more than the base of this handler is called then the
excess is assumed to be a section name and:
:returns: A JSON representation of that particular section of the
:mod:`config`.
:rtype: ``json``
:raises: :exc:`restlite.Status` 404 if the section does not exist.
"""
authentication.login(request)
section = getSection(request['PATH_INFO'])
if section:
settings = config.getSettings(section, False)
if settings is None:
raise restlite.Status, "404 Not Found"
else:
return request.response(settings)
else:
return request.response(config.getSections())
def build(r):
settings = config.getSettings()
# Get the base stylesheet
if 'disable' in settings['stylesheet'] and settings['stylesheet'][
'disable']:
return None
stylesheet = config.getStylesheet()
# Grab the "license" text
header = config.getStylesheetHeader()
# Handle the demonyms, if specified in the settings to do so
if settings['stylesheet']['demonyms_enabled']:
demonyms = getDemonymsCss()
else:
demonyms = ''
# Handle the multiple-headers settings
if settings['stylesheet']['num_of_headers'] > 1:
stylesheet = stylesheet.replace('%%header1%%', getHeader())
# Handle the new header cycle if enabled
if settings['stylesheet']['new_header_cycle']['enabled']:
getNextHeader(r)
# Prepend the demonyms
stylesheet = demonyms + "\n\n" + stylesheet
# Handle the minify stylesheet setting without affecting the header
if settings['stylesheet']['minify']:
stylesheet = cssmin(stylesheet)
# Add the header
stylesheet = header + "\n\n" + stylesheet
return stylesheet
def get(self):
log.log('\tGetting %s streams...' % self.name)
startTime = time()
settings = getSettings()
# Just use the cached copy in dev mode
if settings['dev_mode'] == True:
log.log('\t\t...done getting %s streams! (cached data)' %
self.name)
cachedCopy = cache.readJson('%s.json' % self.name)
return cachedCopy['data'] if cachedCopy != False else []
req = self.makeApiCall(self.api_url)
if req == False:
return self.useCachedCopy()
streams = req[self.streams_field]
ret = []
# Make sure we only have valid live streams, then add them to the
# list of stream objects we're returning
for stream in streams:
streamObj = self.convertStream(stream)
if streamObj != None:
ret.append(streamObj)
elapsedTime = '\BLUE(%s s)' % str(round(time() - startTime, 3))
log.log('\t\t\GREEN...done! %s' % (elapsedTime))
cache.saveJson("%s.json" % self.name, {'time': time(), 'data': ret})
return ret
def POST(request,entity):
authentication.login(request)
groups = config.getSettings("groups")
fields = parseMultipart(request, entity)
if fields is None:
raise restlite.Status, "400 Invalid SUE POST request - need parameters"
# Try to find what we should label the component
component_name = fields.getfirst("component_name")
if not component_name:
raise restlite.Status, "400 Must give the SUE component a name"
# Try to get the SUE component
try:
component_f = fields['component_file']
except KeyError:
raise restlite.Status, "400 Must provide a file when specifying a SUE component"
if component_f.file:
component = component_f.file.read()
saveSueComponent(component_name, component)
else:
raise restlite.Status, "400 The supplied \"component_file\" was not a file"
return request.response("", "text/plain")
def addUser(name, password):
"""
Add a user to the current authentication model.
This will also get update the configuration with the information
needed to recreate itself.
:param name: The name of the user to add.
:type name: ``str``
:param password: The password to use for this user.
:type password: ``str``
:raises: ``KeyError`` if the name exists
"""
global authModel
users = config.getSettings("auth")["users"]
if name in users:
raise KeyError, str(name)+" already exists"
users[name] = password
# Don't need a complete refresh to add one user
if authModel is None:
refresh()
else:
addUserToModel(name, password)
def __applyWorkload(workload_def, target_group):
print("Applying workload")
wkld_dir = os.path.join(
config.getSettings("global")["projdir"],
config.getSettings("workloads")["dir"]
)
args = ['/bin/sh', 'commandline_client.sh']
args.append('--rmi_host='+config.getSettings('command_centre')['rmi_host'])
args.append('--rmi_port='+config.getSettings('command_centre')['rmi_port'])
args.append('--group_rank='+str(target_group['group_number']))
args.append('addworkload')
args.append(workload_def['file'])
subprocess.Popen(args, cwd=vazelsmanager.getVazelsPath()+'/client')
print("Workload dispatched")
def securityDir():
"""
Get the location of our security directory.
This should be somewhere inside our project directory as described
`here <https://github.com/jelford/reef/wiki/Directory-Structure>`_.
:returns: Absolute path to the security directory.
:rtype: ``str``
"""
return os.path.join(
config.getSettings("global")["projdir"],
config.getSettings("security")["dir"],
)
def _extractActors(group_number, actors):
"""
Extract actors into group directories.
:param group_number: The number of the group to extract into.
:type group_number: ``int``
:param actors: The set of actors to extract.
:type actors: ``set(str)``
"""
experiment_dir = controlcentre.getExperimentPath()
for actor_name in actors:
actor = config.getSettings('actors')['defs'][actor_name]
type=actor['type'].upper()
extractpath = os.path.join(
controlcentre.getExperimentPath(),
"Group_"+str(group_number)
)
actorTGZ = tarfile.open(actor['file'],'r:gz')
extractpath = os.path.join(extractpath, "Vazels", type+"_launcher")
actorTGZ.extractall(path=extractpath)
def POST(request, entity):
authentication.login(request)
import urlparse
args = urlparse.parse_qs(entity, True)
# Only accept a single value for any parameter
for key in args:
if len(args[key]) > 1:
raise restlite.Status, "400 Duplicate Arguments"
args[key] = args[key][0]
# Check for action
if "do" not in args:
raise restlite.Status, "400 Missing Action"
# Check for workload
if "workload" not in args:
raise restlite.Status, "400 Missing Workload"
# Check for actor
if "actor" not in args:
raise restlite.Status, "400 Missing Actor"
do = args["do"].lower()
workload = args["workload"]
actor = args["actor"]
# Validate workload
if workload not in config.getSettings("workloads")["defs"]:
raise restlite.Status, "404 Workload Not Found"
# Validate actor
if actor not in config.getSettings("actors")["defs"]:
raise restlite.Status, "404 Actor Not Found (" + actor + ")"
workload = config.getSettings("workloads")["defs"][workload]
try:
{"add": addActor, "rem": remActor}[do](workload, actor)
except KeyError:
raise restlite.Status, "400 Invalid Action"
config.saveConfig()
return request.response(workload)
def POST(request, entity):
authentication.login(request)
section = getSection(request['PATH_INFO'])
# Uncomment below to disallow editing of new sections
#if not section:
# raise restlite.Status, '403 New Section Forbidden'
# Grab arguments
import urlparse
parsed = urlparse.parse_qs(entity, True) # Null value is kept as ''
used_names = [] # Names used
new_qs = {} # Values to add
rem_qs = [] # Values to delete
for key in parsed:
# Grab name and extension
name, ext = splitName(key)
# If name already used, ignore
if name in used_names:
continue
# If theres no extension ignore
if ext is None:
continue
# If there is no value at all, skip
if parsed[key] == []:
continue
# Otherwise take first value
q = parsed[key][0]
# If up for deletion
if ext is 'r':
# Check used_names here as we use key when deleting
# Or parsed name otherwise below
rem_qs += [name]
used_names += [name]
# Adding value
else:
# Try to convert to the correct format
func = funcFor(ext)
if func is None:
raise restlite.Status, "400 Invalid Action"
try:
q = func(q)
except:
raise restlite.Status, "400 Invalid Format"
new_qs[name] = q
used_names += [name]
# All settings are good to go
settings = config.getSettings(section)
# Add all new settings
settings.update(new_qs)
# Delete all new deletions
for key in rem_qs:
try:
del settings[key]
except KeyError: pass
# Make it permanent
config.saveConfig()
return request.response(settings)
def __init__(self):
self.settings = config.getSettings()
self.state = self.settings["state"]
if self.state == "":
self.coord = self.settings["coord"]
if self.coord == "":
import geo
self.coord = geo.getCoord()
def saveSueComponent(component_name, component_file):
import tempfile
sue_dir = os.path.join(
config.getSettings("global")["projdir"],
config.getSettings("SUE")["dir"]
)
ensureDir(sue_dir)
temp_f = tempfile.mkstemp(
suffix='.tar.gz',
prefix='',
dir=sue_dir
)
try:
temp = os.fdopen(temp_f[0], 'wb')
temp.write(component_file)
finally:
temp.close()
newSueComponent = {
"name" : component_name,
"file" : temp_f[1],
}
if component_name in config.getSettings("SUE")["defs"]:
os.remove(config.getSettings("SUE")["defs"][component_name]["file"])
config.getSettings("SUE")["defs"][component_name] = newSueComponent
print config.getSettings("SUE")
config.saveConfig()
def __init__(self):
self.settings = config.getSettings()
request_headers = {}
# Variables to be used in the rest of the object, mainly inherited funcs
self.name = 'Facebook Watch'
self.api_url = ''
self.streams_field = '[]'
self.request_headers = request_headers
def getVazelsPath():
"""
Get the path to the base directory of the Vazels installation.
:returns: Path to the Vazels installation directory.
:rtype: ``str``
"""
return os.path.join(config.getSettings("global")['basedir'],"vazels")
def __init__(self):
settings = config.getSettings()
game_ids = self.convertGames(settings['sidebar']['livestreams']['games'])['MLG']
# The list of games. If there are none, just use CS:GO because CS:GO is awesome.
game_id = ','.join(game_ids) if len(game_ids) > 0 else '13'
self.name = 'MLG'
self.streams_api = ''
self.channels_api = '' % game_id
self.request_headers = {}
def getNextHeader(r):
log.log('> New header cycle is \GREEN ENABLED')
metadata = cache.getMetadata()
settings = config.getSettings()
interval_mins = settings['stylesheet']['new_header_cycle']['interval_mins']
num_next_header = 0
if int(time()) - metadata['new_header_cycle']['last_updated'] > (
interval_mins * 60) - 120:
log.log(
'\t Interval exceeded for current header...grabbing next header')
last_header_name = metadata['new_header_cycle']['last_header']
num_next_header = int(last_header_name.split('.')[0]) + 1
next_header = Path(
'/var/www/globaloffensivebot/public/images/headers/{}.jpg'.format(
str(num_next_header)))
try:
header_exists = next_header.resolve()
except FileNotFoundError:
num_next_header = 1
next_header = '/var/www/globaloffensivebot/public/images/headers/{}.jpg'.format(
str(num_next_header))
with open(
'/var/www/globaloffensivebot/public/images/headers/creators/creators.json',
'r') as f:
creators = json.load(f)
creator = creators[str(num_next_header)]
try:
r.subreddit(settings['subreddit']).stylesheet.upload(
'header', str(next_header))
metadata['new_header_cycle']['last_updated'] = int(time())
metadata['new_header_cycle']['last_header'] = '{}.jpg'.format(
str(num_next_header))
metadata['new_header_cycle']['creator'] = creator
cache.setMetadata(metadata)
except prawcore.TooLarge:
log.error(
'\t\tError uploading {}.jpg by {}. Image too large.'.format(
str(num_next_header), creator))
except praw.exceptions.APIException as e:
print(e)
log.log('\t\t\GREEN...done! \R Now using header #{} by {}\n'.format(
str(num_next_header), creator))
else:
current_num = metadata['new_header_cycle']['last_header'].split('.')[0]
log.log(
'\t Not time to change the current header yet. Using header #{} by {}. Moving on...\n'
.format(current_num, metadata['new_header_cycle']['creator']))
def __init__(self):
settings = config.getSettings()
game_ids = self.convertGames(
settings['sidebar']['livestreams']['games'])['Azubu']
# Just use the first game, if there are none, just use CS:GO because CS:GO is awesome.
# TODO: Implement multiple game searching on Azubu
game_id = game_ids[0] if len(game_ids) > 0 else 'csgo'
limit = str(settings['sidebar']['livestreams']['max_shown'] * 2)
self.name = 'Azubu'
self.api_url = 'http://api.azubu.tv/public/channel/live/list/game/' \
+ game_id + '?limit=' + limit
self.streams_field = 'data'
self.request_headers = {}
def __init__(self):
settings = config.getSettings()
game_ids = self.convertGames(
settings['sidebar']['livestreams']['games'])['Hitbox']
# Just use the first game, if there are none, just use CS:GO because CS:GO is awesome.
# TODO: Implement multiple game searching on Hitbox
game_id = game_ids[0] if len(game_ids) > 0 else '427'
limit = str(settings['sidebar']['livestreams']['max_shown'] * 2)
self.name = 'Hitbox'
self.api_url = 'http://api.hitbox.tv/media/?game=' + game_id \
+ '&live=1&limit=' + limit
self.streams_field = 'livestream'
self.request_headers = {}
def getHeader():
metadata = cache.getMetadata()
numOfHeaders = config.getSettings()['stylesheet']['num_of_headers']
if metadata["header_cycle"]["current_index"] > numOfHeaders:
metadata["header_cycle"]["current_index"] = 0
if int(time()) - int(metadata["header_cycle"]["last_updated"]) > 60 * 60:
# Grabs the new index for the active demonym
metadata["header_cycle"]["current_index"] = (
metadata["header_cycle"]["current_index"] + 1) % numOfHeaders
metadata["header_cycle"]["last_updated"] = int(time())
# Write the metadata object
cache.setMetadata(metadata)
# Returns the Reddit CSS %%variable%%
return "%%header" + str(metadata["header_cycle"]["current_index"] +
1) + "%%"
def buildMarkdown():
log.log('\n> Retrieving community metrics...')
startTime = time()
settings = getSettings()
markdown = ''
if settings['dev_mode'] == True:
log.log('\t... done! (using a cached copy)')
return cache.read('community_metrics.txt')
if settings['sidebar']['social']['ts_enabled']:
teamspeakMd = getTeamspeakUsersMarkdown(settings)
else:
log.log('\t\REDTeamspeak metrics disabled.')
teamspeakMd = None
if settings['sidebar']['social']['irc_enabled']:
ircMd = getIrcUsersMarkdown(settings)
else:
log.log('\t\REDIRC metrics disabled.')
ircMd = None
if settings['sidebar']['social']['discord_enabled']:
disMd = getDiscordUsersMarkdown(settings)
else:
log.log('\t\REDDiscord metrics disabled.')
disMd = None
if teamspeakMd is not None:
markdown += teamspeakMd + '\n'
if ircMd is not None:
markdown += ircMd + '\n'
if disMd is not None:
markdown += disMd
# markdown = teamspeakMd + '\n' + ircMd + '\n' + disMd
# markdown = teamspeakMd + '\n' + ircMd + '\n' + ' '
cache.save('community_metrics.txt', markdown)
elapsedTime = '\BLUE(%s s)' % str(round(time() - startTime, 3))
log.log('\GREENDone retrieving community metrics. %s \n' % elapsedTime)
return markdown
def getAlertExists(verbose=False):
"""
lets see if our alerts have any of the keywords we are looking for
"""
settings = config.getSettings()
keyWords = settings["alertEventTypeKeyWords"]
alerts = getAlerts()
flag = False
for word in keyWords:
if alerts:
for alert in alerts:
alertType = alerts[alert]["eventType"]
if operator.contains(alertType, word):
flag = True
if verbose:
return {"alertExists": flag, "Raw Message": alerts}
else:
return {"alertExists": flag}
def buildMarkdown():
log.log("> Building calendar...")
settings = getSettings()
events = getEvents(settings['sidebar']['calendar'])
if len(events) == 0:
return ''
markdown = '[*Upcoming Events*](#heading)\n\n. | .\n---|---\n'
for event in events:
date = datetime.datetime.strptime(event['start']['date'], '%Y-%m-%d')
link = event['htmlLink']
markdown += date.strftime('%b **%-d**') + ' | ' + '[{}]'.format(
event['summary']) + '({})'.format(link) + '\n'
markdown += '\n**[See all](http://goo.gl/q9iOZb#button#slim)**'
return markdown
def finish():
settings = config.getSettings()
if 'db' not in settings or 'mongo' not in settings['db']:
return
# Send the HTML version of the log to Mongo for webpanel access
logContentsHtml = file_manager.readAndDelete('app-cache/log-html.txt')
# Establish database connection
client = MongoClient(settings['db']['mongo']['host'],
settings['db']['mongo']['port'])
db = client[settings['db']['mongo']['database']]
db.authenticate(settings['db']['mongo']['username'],
settings['db']['mongo']['password'],
mechanism='SCRAM-SHA-1')
# Insert the log as a document, including a timestamp
db['botlog'].insert_one({
"body": logContentsHtml,
"time_finished": str(int(time()))
})
def get(self):
log.log('\tGetting %s streams...' % self.name)
startTime = time()
settings = config.getSettings()
if settings['dev_mode'] == True:
log.log('\t\t...done getting %s streams! (cached data)' % self.name)
cachedCopy = cache.readJson('%s.json' % self.name)
return cachedCopy['data'] if cachedCopy != False else []
# Get all CS:GO streams (CS:GO's MLG game ID is 13)
mlgCsgoStreams = self.makeApiCall(self.channels_api)
if mlgCsgoStreams == False:
return self.useCachedCopy()
streams = {}
# Convert streams to usable, more uniform format
for stream in mlgCsgoStreams:
streams[stream['stream_name']] = self.convertStream(stream)
# Get viewer info on all CS:GO channels
streamInfo = self.makeApiCall(self.streams_api)
if streamInfo == False:
return self.useCachedCopy()
liveStreams = []
for stream in streamInfo:
if stream['stream_name'] in streams:
if 'viewers' in stream:
streams[stream['stream_name']]['viewers'] = '{:,}'.format(stream['viewers'])
streams[stream['stream_name']]['viewers_raw'] = stream['viewers']
liveStreams.append(streams[stream['stream_name']])
elapsedTime = '\BLUE(%s s)' % str(round(time() - startTime, 3))
log.log('\t\t\GREEN...done! %s' % (elapsedTime))
cache.saveJson("%s.json" % self.name, {'time': time(), 'data': liveStreams})
return liveStreams
def getStatus():
log.log('Getting CS:GO Matchmaking status...')
startTime = time()
settings = getSettings()
if settings['dev_mode'] == True:
log.log('\t... done! (using a cached copy)')
return cache.readJson('matchmaking.json')
if 'api_key' not in settings or 'steam' not in settings['api_key']:
log.error('No Steam API key -- cannot retrieve CS:GO server status.')
return {'status': 'UNKNOWN', 'url': 'down'}
offline = {'status': 'OFFLINE', 'url': 'down'}
online = {'status': 'ONLINE', 'url': 'up'}
try:
api_url = 'https://api.steampowered.com/ICSGOServers_730/GetGameServersStatus/v1/?key=%s'
req = requests.get(api_url % settings['api_key']['steam'])
except Exception as e:
elapsedTime = '\BLUE(%s s)' % str(round(time() - startTime, 3))
log.error(
'Could not retrieve CS:GO GC status. (%s) %s' %
(str(e), elapsedTime), 1)
return offline
status = req.json()['result']['matchmaking']['scheduler']
elapsedTime = '\BLUE(%s s)' % str(round(time() - startTime, 3))
log.log('\t...done! %s' % elapsedTime)
if status == 'offline':
cache.saveJson('matchmaking.json', offline)
return offline
elif status == 'normal':
cache.saveJson('matchmaking.json', online)
return online
else:
return {'status': 'UNKNOWN (%s)' % status, 'url': 'down'}
def __init__(self):
self.settings = config.getSettings()
self.oauth_info = config.getOAuthInfo()
self.is_token_valid = self.validateToken(self.oauth_info)
self.token = self.getOauth()
parameters = ''
request_headers = {}
game_id = 0
max_shown = str(self.settings['sidebar']['livestreams']['max_shown'] *
2)
if 'twitch' in self.settings and 'client_id' in self.settings['twitch']:
request_headers = {
'Authorization': 'Bearer {}'.format(self.token),
'Client-ID': self.settings['twitch']['client_id']
}
# If the sidebar.livestreams.twitch_include_only field is set in
# settings, then we only grab those channels from Twitch.
if 'twitch_include_only' in self.settings['sidebar']['livestreams']:
channels = self.settings['sidebar']['livestreams'][
'twitch_include_only']
if channels != []:
parameters.append('channel=' + ','.join(channels))
if 'game_ids' in self.settings['sidebar']['livestreams']:
if 'CSGO' in self.settings['sidebar']['livestreams']['game_ids']:
game_id = self.settings['sidebar']['livestreams']['game_ids'][
'CSGO']
# Turn parameters list into a stringified URL parameter chain
parameters = '?game_id={}&first={}'.format(game_id, max_shown)
# Variables to be used in the rest of the object, mainly inherited funcs
self.name = 'Twitch'
self.api_url = 'https://api.twitch.tv/helix/streams/' + parameters
self.streams_field = 'data'
self.request_headers = request_headers
def getShortUrl(longUrl):
settings = getSettings()
if 'api_key' not in settings or 'google' not in settings['api_key']:
log.error('No Google API key -- cannot shorten URLs')
return longUrl
data = json.dumps({'longUrl': longUrl})
headers = {'Content-Type': 'application/json'}
endpoint = 'https://www.googleapis.com/urlshortener/v1/url?key=%s' % settings[
'api_key']['google']
shortUrl = 'http://example.org/'
try:
req = requests.post(endpoint, data=data, headers=headers)
except Exception as e:
log.error('Could not shorten URL (%s)\n- %s' % (str(e), longUrl), 1)
return longUrl
try:
shortUrl = req.json()['id']
except Exception as e:
log.error('Error parsing Google API JSON: %s' % str(e), 1)
return longUrl
return shortUrl
def handle_database(command):
connection = None
try:
config_data = getDbConfig(config.getSettings())
connect_str = "dbname='" + config_data[
'db_name'] + "' user='******'user'] + "' host='" + config_data[
'host'] + "' password='******'password'] + "'"
connection = psycopg2.connect(connect_str)
connection.autocommit = True
cursor = connection.cursor()
cursor.execute(command)
if "SELECT" in command:
table = cursor.fetchall()
return table
cursor.close()
except psycopg2.DatabaseError as exception:
error_message = "Uh oh, can't connect. Invalid dbname, user or password? \n" + str(
exception)
print(error_message)
finally:
if connection:
connection.close()
def buildMarkdown():
log.log('> Beginning to build the matchticker...')
startTime = time()
settings = getSettings()
if settings['dev_mode'] == True:
log.log('\t...done! (using a cached copy)')
return cache.read('matchticker.txt')
if 'api_key' not in settings or 'gosugamers' not in settings['api_key']:
log.error('No GosuGamers API key -- cannot build matchticker.')
return ''
# Get the stream information
try:
api_url = ''
req = requests.get(api_url % settings['api_key']['gosugamers'])
except requests.exceptions.RequestException as e:
elapsedTime = '\BLUE(%s s)' % str(round(time() - startTime, 3))
log.error('From GosuGamers API: %s %s' % (str(e), elapsedTime), 1)
return ''
if req.status_code == 403 or not req.ok or 'IP Address Not Allowed' in str(
req.content):
elapsedTime = '\BLUE(%s s)' % str(round(time() - startTime, 3))
log.error('GosuGamers rejected our IP ' + elapsedTime, 1)
return blankTicker(startTime)
try:
upcomingMatches = req.json()['matches']
except Exception as e:
elapsedTime = '\BLUE(%s s)' % str(round(time() - startTime, 3))
log.error(
'Issue with GosuGamers API JSON: %s %s' % (str(e), elapsedTime), 1)
return ''
# Matches to display
matches = []
gamesToGrab = 0
if len(upcomingMatches) == 0:
return blankTicker(startTime)
if len(upcomingMatches) < settings['sidebar']['matchticker']['max_shown']:
gamesToGrab = len(upcomingMatches)
else:
gamesToGrab = settings['sidebar']['matchticker']['max_shown']
for i in range(0, gamesToGrab):
matches.append({
'tourny':
prepareTournyTitle(upcomingMatches[i]['tournament']['name']),
'team1': {
'name':
str(upcomingMatches[i]['firstOpponent']['shortName']),
'cc':
str(upcomingMatches[i]['firstOpponent']['country']
['countryCode']).lower()
},
'team2': {
'name':
str(upcomingMatches[i]['secondOpponent']['shortName']),
'cc':
str(upcomingMatches[i]['secondOpponent']['country']
['countryCode']).lower()
},
'time':
getMatchTime(upcomingMatches[i]['datetime']),
'url':
upcomingMatches[i]['pageUrl'],
'is_live':
bool(upcomingMatches[i]["isLive"])
})
# Build the markdown
matchtickerMd = ''
matchMdTemplate = ('>>>\n'
'[~~__TOURNY__~~\n'
'~~__TIME__~~\n'
'~~__TEAM1__~~\n'
'~~__TEAM2__~~](__URL__#info)\n'
'[ ](#lang-__LANG1__)\n'
'[ ](#lang-__LANG2__)\n\n'
'>>[](#separator)\n\n')
matchtickerMd = '[*Match Ticker*](#heading)\n\n'
i = 0
for match in matches:
matchMd = matchMdTemplate
matchMd = (matchMd.replace('__TOURNY__', match['tourny']).replace(
'__TIME__',
match['time']).replace('__TEAM1__',
match['team1']['name']).replace(
'__TEAM2__',
match['team2']['name']).replace(
'__LANG1__',
match['team1']['cc']).replace(
'__LANG2__',
match['team2']['cc']).replace(
'__URL__', match['url']))
matchtickerMd += matchMd
i += 1
matchtickerMd += '>>**[See all](http://bit.ly/1xGEuiJ#button#slim)**'
cache.save('matchticker.txt', matchtickerMd)
characters = '\YELLOW(%d characters)' % len(matchtickerMd)
elapsedTime = '\BLUE(%s s)' % str(round(time() - startTime, 3))
log.log('\t\GREEN...done! %s %s \n' % (characters, elapsedTime))
return matchtickerMd
def build():
log.log('> Beginning to build the livestream feed...')
livestreamFeedStartTime = time()
settings = config.getSettings()
# Fetch all the livestream data
livestreams = []
services = (s.lower()
for s in settings['sidebar']['livestreams']['services'])
if settings['sidebar']['livestreams']['services'] != []:
if 'twitch' in services:
try:
livestreams += twitch.Twitch().get()
except KeyError as ke:
log.log(
'\t\tFailed to retrieve Twitch streams. \n\t\tKeyError: {}'
.format(ke))
if 'youtube' in services:
try:
livestreams += youtube.youtube_search()
except KeyError as ke:
log.log(
'\t\tFailed to retrieve YouTube Gaming streams. \n\t\tKeyError: {}'
.format(ke))
if 'fbwatch' in services:
try:
livestreams += fbwatch.FBWatch().get()
except KeyError as ke:
log.log(
'\t\tFailed to retrieve Facebook Watch ESL stream. \n\t\tKeyError: {}'
.format(ke))
else:
livestreams = twitch.Twitch().get() \
+ hitbox.Hitbox().get() \
+ azubu.Azubu().get() \
+ mlg.MLG().get() \
+ youtube.youtube_search() \
+ fbwatch.FBWatch().get()
# Return a blank container's markdown if there are no streams
if len(livestreams
) == 0 and settings['sidebar']['livestreams']['none_message'] != '':
return {
'markdown': '[*Live Streams*](#heading)\n\n %s' \
% settings['sidebar']['livestreams']['none_message'],
'spritesheet_path': None
}
# Sort the livestreams by number of viewers, descending
livestreams = sorted(livestreams,
key=lambda channel: channel['viewers_raw'],
reverse=True)
# Trim off the remainder if there were more than the number we're supposed to use
if len(livestreams) > settings['sidebar']['livestreams']['max_shown']:
livestreams = livestreams[:settings['sidebar']['livestreams']
['max_shown']]
# TODO: Clean up all this code, the naming is all horrid
templates = config.getTemplates()
# Goes in between items in the livestream list
if 'livestreams' in templates and 'separator' in templates['livestreams']:
separator = templates['livestreams']['separator']
else:
separator = '>>[](#separator)'
separator = '\n\n' + separator + '\n\n'
# Template for a stream in the livestream list
if 'livestreams' in templates and 'stream' in templates['livestreams']:
livestreamMdTemplate = templates['livestreams']['stream']
else:
# Default template
livestreamMdTemplate = ('>>>#[__TITLE__](__URL__#profile-__INDEX__)\n'
'##\n'
'### __VIEWERS__ @ __STREAMER__\n')
livestreamMdTemplate += separator
# Template for the livestreams section heading
if 'livestreams' in 'templates' and 'heading' in templates['livestreams']:
livestreamsMd = templates['livestreams']['heading'] + '\n\n'
else:
# Default heading
livestreamsMd = '[*Live Streams*](#heading)\n\n'
i = 0
for livestream in livestreams:
livestreamMd = livestreamMdTemplate
livestreamMd = (livestreamMd.replace(
'__TITLE__',
livestream['title']).replace('__URL__', livestream['url']).replace(
'__INDEX__',
str(i)).replace('__VIEWERS__', livestream['viewers']).replace(
'__STREAMER__', livestream['streamer']))
livestreamsMd += livestreamMd
i += 1
if 'see_all_link' in settings['sidebar']['livestreams']:
if 'livestreams' in templates and 'see_all' in templates['livestreams']:
see_all_template = templates['livestreams']['see_all']
else:
see_all_template = '>>**[See all](__LINK__#button#slim)**'
see_all_template = see_all_template.replace(
'__LINK__', settings['sidebar']['livestreams']['see_all_link'])
livestreamsMd += see_all_template
# This ignore bit is because any lines starting with a hashtag
# are headers in Reddit Markdown and comments in YAML, so in the
# YAML, these lines must be preceded with something unique so that
# we can parse those lines properly. In case this was pulled from the
# wiki, we need to remove all the potential %%IGNORE%%s from the string
livestreamsMd = livestreamsMd.replace('%%IGNORE%%', '')
spritesheetPath = None
if settings['sidebar']['livestreams']['show_thumbnails']:
uploadThumbnailsStartTime = time()
log.log('\n\tGenerating spritesheet...')
spritesheetPath = generateSpritesheet(
[x['thumbnail'] for x in livestreams])
log.log('\t\t\GREEN...done! ' + '\BLUE(%s s)' %
str(round(time() - uploadThumbnailsStartTime, 3)))
characters = '\YELLOW(%d characters)' % len(livestreamsMd)
elapsedTime = '\BLUE(%s s)' % str(
round(time() - livestreamFeedStartTime, 3))
log.log('\GREENDone building livestream feed. %s %s \n' %
(characters, elapsedTime))
return {'markdown': livestreamsMd, 'spritesheet_path': spritesheetPath}
import config
import log
from time import time
from apiclient import discovery
from apiclient import errors
from oauth2client.tools import argparser
from pprint import pprint
settings = config.getSettings()
DEVELOPER_KEY = settings['api_key']['youtube']
YOUTUBE_API_SERVICE_NAME = "youtube"
YOUTUBE_API_VERSION = "v3"
SEARCH_PART = "snippet"
VIDEOS_PART = "liveStreamingDetails"
base_url = 'https://gaming.youtube.com/watch?v='
ret = []
def youtube_search():
log.log('\tGetting YouTube Gaming streams...')
startTime = time()
youtube = discovery.build(YOUTUBE_API_SERVICE_NAME,
YOUTUBE_API_VERSION,
developerKey=DEVELOPER_KEY)
for channel in settings['sidebar']['livestreams'][
'youtube_channels'].values():
try:
# Call search.list to retrieve results matching the specified query term
def runAutoposterAndBuildMarkdown(r):
notices = config.getNotices()
settings = config.getSettings()
noticesToBuild = []
potentialStickies = []
# Extra \n just to add some whitespace
log.log('> Running the auto-poster and building notices...')
startTime = time()
for notice in notices:
if notice['notice_title'] != 'Default Notice Title':
log.log("\t\MAGENTAFor \"%s\"" % notice['notice_title'])
else:
log.log("\t\MAGENTAFor \"%s\"" % notice['thread_title'])
live = '\GREENlive'
notLive = '\REDnot live'
postIsLive = isPostLive(notice)
log.log('\t\tThe post is ' + (live if postIsLive else notLive) + '.')
noticeIsLive = isNoticeLive(notice)
log.log('\t\tThe notice is ' + (live if noticeIsLive else notLive) +
'.')
if noticeIsLive:
noticesToBuild.append(notice)
# Determine eligibility for being posted
# --- Subtracting 60 sec from the last_posted time is to account for the fact that no
# --- notices/threads are ever posted at *exactly* the scheduled time, because it takes a few
# --- seconds to post at the least. Therefore, without shaving off 60 seconds (which is a very
# --- generous 'buffer' for how long it may take to post any given recurring notice/thread),
# --- the 'last_posted' time will increase by a few seconds each week, making a scheduled item
# --- not go up until the next 5 min update interval, and continues increasing until 'Reset Timing'
# --- is issued from the webpanel. This is really just a bandaid for now and should be rewritten.
timeSinceLastPosted = (int(time()) - notice['last_posted']) - 60
eligibleForPosting = timeSinceLastPosted > frequency[
notice['frequency']]
if notice['frequency'] == 'once' and notice['last_posted'] != 0:
eligibleForPosting = False
# Determine if it should be posted, and get its ID
if eligibleForPosting and postIsLive:
post = createPostFromNotice(r, settings, notice)
notice['last_posted'] = int(time())
notice['last_posted_id'] = post.id
post_id = post.id
else:
post_id = notice['last_posted_id']
# Handle stickies later when we can ensure we're on the primary bot acct
if post_id != None and post_id != "":
potentialStickies.append({'postIsLive': postIsLive, 'id': post_id})
log.log('')
# Handle whether or not the posts should be stickied
for potential in potentialStickies:
stickied = isPostStickied(r, potential['id'])
if stickied == False and potential['postIsLive']:
stickyPost(r, potential['id'])
elif stickied and potential['postIsLive'] == False:
unstickyPost(r, potential['id'])
# Write changes of the notices to the notices file
config.saveNotices(notices)
noticesMd = ''
# Shuffle the notices if necessary
if len(noticesToBuild) > 3:
random.shuffle(noticesToBuild)
noticesToBuild = noticesToBuild[:3]
# Build the notices markdown string
template = "1. [__TITLE__](__LINK__#__CATEGORY__)\n"
for notice in noticesToBuild:
if notice['type'] == 'autopost+notice':
noticesMd += (template.replace(
'__TITLE__', formatTitle(notice['notice_title'])).replace(
'__LINK__',
"http://redd.it/" + notice['last_posted_id']).replace(
'__CATEGORY__', notice['category']))
else:
noticesMd += (template.replace(
'__TITLE__', formatTitle(notice['notice_title'])).replace(
'__LINK__',
notice['notice_link']).replace('__CATEGORY__',
notice['category']))
elapsedTime = '\BLUE(%s s)' % str(round(time() - startTime, 3))
log.log('\GREENDone running the auto-poster and building notices! %s \n' %
elapsedTime)
# Return the markdown
return noticesMd
# here comes a check if server is prod
is_prod = False
if is_dev:
return db_settings['dev']
elif is_test:
return db_settings['test']
elif is_prod:
return db_settings['prod']
return dict()
try:
# get config data
config_data = getDbConfig(config.getSettings())
# setup connection string
connect_str = "dbname='" + config_data[
'db_name'] + "' user='******'user'] + "' host='" + config_data[
'host'] + "' password='******'password'] + "'"
# use our connection values to establish a connection
conn = psycopg2.connect(connect_str)
# set autocommit option, to do every query when we call it
conn.autocommit = True
# create a psycopg2 cursor that can execute queries
cursor = conn.cursor()
# removing the test table if it already exists
cursor.execute("""DROP TABLE IF EXISTS connection_check;""")
# create a new table with a single column called "name"
cursor.execute("""CREATE TABLE connection_check (name varchar(40));""")
def __init__(self):
self.settings = config.getSettings()
self.weatherAPIKey = self.settings["openWeatherMapAPIKey"]
self.zipCode = self.settings["zipCode"]
self.units = self.settings["units"]
