def __init__(self):
fs_to_data_lake: Mapping[str, Callable[[], Tuple[
BaseLandingArea, BlobStagingArea, BlobPublicArea]]] = {
FileSystemType.blob: self.__get_blob_data_lakes,
FileSystemType.local: self.__get_local_data_lakes,
}
fs_to_web_area: Mapping[str, Type[BaseWebArea]] = {
FileSystemType.blob: BlobWebArea,
FileSystemType.local: LocalWebArea,
}
self.landing, self.staging, self.public = fs_to_data_lake[
Config().file_system_type]()
self.web = fs_to_web_area[Config().web_config.fs](
**Config().web_config.attrs)
self.big_query = BigQuery(project=Config().bq_project,
client_secrets=Config().bq_secret)
def generate_email_body(date: datetime, company=Config().default_company):
report = OSCIChangeRanking(date=date)
company_contributors_ranking_schema = DataLake(
).public.schemas.company_contributors_ranking
change_ranking = report.read().reset_index()
change_ranking = change_ranking.rename(
columns={'index': company_contributors_ranking_schema.position})
change_ranking[company_contributors_ranking_schema.position] += 1
change_ranking = __cast_columns_to_int(
df=change_ranking,
columns=[
report.schema.total,
report.schema.active,
company_contributors_ranking_schema.position,
report.schema.total_change,
report.schema.active_change,
report.schema.position_change,
])
shift_up = __add_arrows_prefix(df=__get_shift_up(
change_ranking=change_ranking,
change_position_field=report.schema.position_change),
column=report.schema.position_change)
shift_down = __add_arrows_prefix(df=__get_shift_down(
change_ranking=change_ranking,
change_position_field=report.schema.position_change),
column=report.schema.position_change)
company_position = __add_arrows_prefix(
df=__get_company_neighbors(
df=change_ranking,
company=company,
company_field=report.schema.company,
rank_field=company_contributors_ranking_schema.position),
column=report.schema.position_change)
DataLake().public.save_email(email_body=EmailBodyTemplate().render(
date=date,
compared_date=get_previous_date(date),
shift_up=shift_up,
shift_down=shift_down,
company=company,
company_position=company_position,
solutionshub_osci_change_ranking=OSCIChangeRankingExcel(
to_date=date).url,
osci_reports_urls={
name: report_cls(date=date).url
for name, report_cls in OSCI_REPORTS_URLS.items()
}),
date=date)
def _repositories(names: Iterable[str]):
with GithubRest(token=Config().github_token) as rest:
for name in names:
try:
repo_resp = rest.get_repository(repo_name=name)
if repo_resp is not None:
repository = parse_get_repository_response(
repo_resp, downloaded_at=datetime.now().date())
if repository.__getattribute__(
Repositories.schema.license):
yield {
field: repository.__getattribute__(field)
for field in Repositories.schema.required
}
except Exception as ex:
log.error(f'Failed loading repository {name}. ex: {ex}')
def load_company_repositories_events_commits(date: datetime, company: str):
events = DataLake().staging.get_push_events_commits(
company=company,
from_date=date,
to_date=date,
date_period_type=DatePeriodType.DTD)
schema = DataLake().staging.schemas.push_commits
if events.empty:
log.warning(f'No {company} events at {date}')
return
with GithubRest(token=Config().github_token) as rest:
company_commits = get_company_repositories_events_commits(
repositories_names=events[schema.repo_name].unique(),
date=date,
company=company,
rest=rest)
company_commits_df = pd.DataFrame(company_commits)
DataLake().staging.save_private_push_events_commits(
push_event_commits=company_commits_df,
company_name=company,
date=date)
def runClient(options):
# log initialization
log.message("Starting Outer Space Client", ige.version.versionString)
log.debug("sys.path =", sys.path)
log.debug("os.name =", os.name)
log.debug("sys.platform =", sys.platform)
log.debug("os.getcwd() =", os.getcwd())
log.debug("sys.frozen =", getattr(sys, "frozen", None))
# create required directories
if not os.path.exists(options.configDir):
os.makedirs(options.configDir)
log.debug("options.configDir =", options.configDir)
running = 1
first = True
#while running:
if not first:
reload(osci)
# parse configuration
if first:
import osci.gdata as gdata
else:
reload(gdata)
gdata.config = Config(
os.path.join(options.configDir, options.configFilename))
gdata.config.game.server = options.server
setDefaults(gdata, options)
language = gdata.config.client.language
import gettext
log.debug('OSCI', 'Installing translation for:', language)
if language == 'en':
log.debug('OSCI', 'English is native - installing null translations')
tran = gettext.NullTranslations()
else:
try:
tran = gettext.translation('OSPACE',
resources.get('translations'),
languages=[language])
except IOError:
log.warning('OSCI', 'Cannot find catalog for', language)
log.message('OSCI', 'Installing null translations')
tran = gettext.NullTranslations()
tran.install(unicode=1)
#initialize pygame and prepare screen
if (gdata.config.defaults.sound == "yes") or (gdata.config.defaults.music
== "yes"):
pygame.mixer.pre_init(44100, -16, 2, 4096)
os.environ['SDL_VIDEO_ALLOW_SCREENSAVER'] = '1'
os.environ['SDL_DEBUG'] = '1'
pygame.init()
flags = pygame.SWSURFACE
DEFAULT_SCRN_SIZE = (800, 600)
gdata.scrnSize = DEFAULT_SCRN_SIZE
if gdata.config.display.resolution == "FULLSCREEN":
gdata.scrnSize = (0, 0)
flags |= pygame.FULLSCREEN
elif gdata.config.display.resolution is not None:
width, height = gdata.config.display.resolution.split('x')
gdata.scrnSize = (int(width), int(height))
if gdata.config.display.depth == None:
# guess best depth
bestdepth = pygame.display.mode_ok(gdata.scrnSize, flags)
else:
bestdepth = int(gdata.config.display.depth)
# initialize screen
try:
screen = pygame.display.set_mode(gdata.scrnSize, flags, bestdepth)
# gdata.scrnSize is used everywhere to setup windows
gdata.scrnSize = screen.get_size()
except pygame.error:
# for example if fullscreen is selected with resolution bigger than display
# TODO: as of now, fullscreen has automatic resolution
gdata.scrnSize = DEFAULT_SCRN_SIZE
screen = pygame.display.set_mode(gdata.scrnSize, flags, bestdepth)
gdata.screen = screen
log.debug('OSCI', 'Driver:', pygame.display.get_driver())
log.debug('OSCI', 'Using depth:', bestdepth)
log.debug('OSCI', 'Display info:', pygame.display.Info())
pygame.mouse.set_visible(1)
pygame.display.set_caption(_('Outer Space %s') % ige.version.versionString)
# set icon
pygame.display.set_icon(
pygame.image.load(resources.get('icon48.png')).convert_alpha())
# UI stuff
if first:
import pygameui as ui
else:
reload(ui)
setSkinTheme(gdata, ui)
app = ui.Application(update, theme=ui.SkinableTheme)
app.background = defineBackground()
app.draw(gdata.screen)
app.windowSurfaceFlags = pygame.SWSURFACE | pygame.SRCALPHA
gdata.app = app
pygame.event.clear()
# resources
import osci.res
osci.res.initialize()
# load resources
import osci.dialog
dlg = osci.dialog.ProgressDlg(gdata.app)
osci.res.loadResources(dlg)
dlg.hide()
osci.res.prepareUIIcons(ui.SkinableTheme.themeIcons)
while running:
if first:
import osci.client, osci.handler
from igeclient.IClient import IClientException
else:
reload(osci.client)
reload(osci.handler)
osci.client.initialize(gdata.config.game.server, osci.handler, options)
# create initial dialogs
if first:
import osci.dialog
else:
reload(osci.dialog)
gdata.savePassword = gdata.config.game.lastpasswordcrypted != None
if options.login and options.password:
gdata.config.game.lastlogin = options.login
gdata.config.game.lastpassword = options.password
gdata.config.game.lastpasswordcrypted = binascii.b2a_base64(
options.password).strip()
gdata.config.game.autologin = '******'
gdata.savePassword = '******'
loginDlg = osci.dialog.LoginDlg(gdata.app)
updateDlg = osci.dialog.UpdateDlg(gdata.app)
# event loop
update()
lastSave = time.clock()
# set counter to -1 to trigger Update dialog (see "if" below)
counter = -1
needsRefresh = False
session = 1
first = False
while running and session:
try:
counter += 1
if counter == 0:
# display initial dialog in the very first cycle
updateDlg.display(caller=loginDlg, options=options)
# process as many events as possible before updating
evt = pygame.event.wait()
evts = pygame.event.get()
evts.insert(0, evt)
forceKeepAlive = False
saveDB = False
for evt in evts:
if evt.type == pygame.QUIT:
running = 0
break
if evt.type == (
ui.USEREVENT) and evt.action == "localExit":
session = False
break
if evt.type == pygame.ACTIVEEVENT:
if evt.gain == 1 and evt.state == 6:
# pygame desktop window focus event
needsRefresh = True
if evt.type == pygame.KEYUP and evt.key == pygame.K_F12:
if not pygame.key.get_mods() & pygame.KMOD_CTRL:
running = 0
break
if evt.type == pygame.KEYUP and evt.key == pygame.K_F9:
forceKeepAlive = True
evt = gdata.app.processEvent(evt)
if gdata.app.needsUpdate() or needsRefresh:
needsRefresh = False
update()
# keep alive connection
osci.client.keepAlive(forceKeepAlive)
# save DB every 4 hours in case of a computer crash
# using "counter" to limit calls to time.clock() to approximately every 10-15 minutes
if counter > 5000:
# set this to zero so we don't display Update dialog
counter = 0
if time.clock() - lastSave > 14400:
saveDB = True
if saveDB:
osci.client.saveDB()
lastSave = time.clock()
except IClientException, e:
osci.client.reinitialize()
gdata.app.setStatus(e.args[0])
loginDlg.display(message=e.args[0])
except Exception, e:
log.warning('OSCI', 'Exception in event loop')
if not isinstance(e, SystemExit) and not isinstance(
e, KeyboardInterrupt):
log.debug("Processing exception")
# handle exception
import traceback, StringIO
fh = StringIO.StringIO()
exctype, value, tb = sys.exc_info()
funcs = [entry[2] for entry in traceback.extract_tb(tb)]
faultID = "%06d-%03d" % (
hash("/".join(funcs)) % 1000000,
traceback.extract_tb(tb)[-1][1] % 1000,
)
del tb
# high level info
print >> fh, "Exception ID:", faultID
print >> fh
print >> fh, "%s: %s" % (exctype, value)
print >> fh
print >> fh, "--- EXCEPTION DATA ---"
# dump exception
traceback.print_exc(file=fh)
excDlg = osci.dialog.ExceptionDlg(gdata.app)
excDlg.display(faultID, fh.getvalue())
del excDlg # reference to the dialog holds app's intance
fh.close()
del fh
else:
break
def __get_blob_data_lakes(
) -> Tuple[BlobLandingArea, BlobStagingArea, BlobPublicArea]:
return (BlobLandingArea(**Config().file_system.landing_props),
BlobStagingArea(**Config().file_system.staging_props),
BlobPublicArea(**Config().file_system.public_props))
def __get_local_data_lakes(
) -> Tuple[LocalLandingArea, LocalStagingArea, LocalPublicArea]:
return (LocalLandingArea(**Config().file_system.landing_props),
LocalStagingArea(**Config().file_system.staging_props),
LocalPublicArea(**Config().file_system.public_props))
import ige.ospace.Rules as Rules
Rules.init(options.configDir)
from osci.StarMap import StarMap
import resources
import osci.res
import pygame, pygame.ftfont, pygame.freetype
import pygameui, pygameui.SkinableTheme
from osci.config import Config
import osci.gdata as gdata
import osci.client
import ige.version
import gettext
import hashlib
gettext.NullTranslations().install(unicode=1)
gdata.config = Config(
os.path.join(options.configDir, options.configFilename))
gdata.config.game.server = 'localhost:9080'
gdata.config.defaults.minfleetsymbolsize = 4
gdata.config.defaults.minplanetsymbolsize = 5
gdata.config.defaults.maxfleetsymbolsize = 0
gdata.config.defaults.maxplanetsymbolsize = 0
options.heartbeat = 60
osci.client.initialize('localhost:9080', dummyHandler, options)
osci.client.login(options.game, 'admin', password)
osci.client.cmdProxy.selectAdmin()
osci.client.updateDatabase()
pygame.init()
def test_config_singleton():
config1 = Config()
config2 = Config()
Config.tear_down()
assert id(config1) == id(config2)
import logging
from datetime import datetime
import click
logging.basicConfig(format='[%(asctime)s] [%(levelname)s] %(message)s',
level=logging.DEBUG)
log = logging.getLogger(__name__)
if 'dbutils' in globals():
log.debug(
'Variable `dbutils` in memory. Try to setup config with `dbutils`')
from osci.config import Config, FileSystemType
config = Config(dbutils=dbutils)
if 'spark' in globals():
log.debug('Variable `spark` in memory.')
from osci.jobs.session import Session
log.debug('Get spark session')
Session(spark_session=spark)
if Config().file_system_type == FileSystemType.blob:
spark.conf.set(
f'fs.azure.account.key.{Config().file_system.staging_props.get("storage_account_name")}.'
f'blob.core.windows.net',
Config().file_system.staging_props.get('storage_account_key'))
from osci.actions import Action