def __init__(self): fs_to_data_lake: Mapping[str, Callable[[], Tuple[ BaseLandingArea, BlobStagingArea, BlobPublicArea]]] = { FileSystemType.blob: self.__get_blob_data_lakes, FileSystemType.local: self.__get_local_data_lakes, } fs_to_web_area: Mapping[str, Type[BaseWebArea]] = { FileSystemType.blob: BlobWebArea, FileSystemType.local: LocalWebArea, } self.landing, self.staging, self.public = fs_to_data_lake[ Config().file_system_type]() self.web = fs_to_web_area[Config().web_config.fs]( **Config().web_config.attrs) self.big_query = BigQuery(project=Config().bq_project, client_secrets=Config().bq_secret)
def generate_email_body(date: datetime, company=Config().default_company): report = OSCIChangeRanking(date=date) company_contributors_ranking_schema = DataLake( ).public.schemas.company_contributors_ranking change_ranking = report.read().reset_index() change_ranking = change_ranking.rename( columns={'index': company_contributors_ranking_schema.position}) change_ranking[company_contributors_ranking_schema.position] += 1 change_ranking = __cast_columns_to_int( df=change_ranking, columns=[ report.schema.total, report.schema.active, company_contributors_ranking_schema.position, report.schema.total_change, report.schema.active_change, report.schema.position_change, ]) shift_up = __add_arrows_prefix(df=__get_shift_up( change_ranking=change_ranking, change_position_field=report.schema.position_change), column=report.schema.position_change) shift_down = __add_arrows_prefix(df=__get_shift_down( change_ranking=change_ranking, change_position_field=report.schema.position_change), column=report.schema.position_change) company_position = __add_arrows_prefix( df=__get_company_neighbors( df=change_ranking, company=company, company_field=report.schema.company, rank_field=company_contributors_ranking_schema.position), column=report.schema.position_change) DataLake().public.save_email(email_body=EmailBodyTemplate().render( date=date, compared_date=get_previous_date(date), shift_up=shift_up, shift_down=shift_down, company=company, company_position=company_position, solutionshub_osci_change_ranking=OSCIChangeRankingExcel( to_date=date).url, osci_reports_urls={ name: report_cls(date=date).url for name, report_cls in OSCI_REPORTS_URLS.items() }), date=date)
def _repositories(names: Iterable[str]): with GithubRest(token=Config().github_token) as rest: for name in names: try: repo_resp = rest.get_repository(repo_name=name) if repo_resp is not None: repository = parse_get_repository_response( repo_resp, downloaded_at=datetime.now().date()) if repository.__getattribute__( Repositories.schema.license): yield { field: repository.__getattribute__(field) for field in Repositories.schema.required } except Exception as ex: log.error(f'Failed loading repository {name}. ex: {ex}')
def load_company_repositories_events_commits(date: datetime, company: str): events = DataLake().staging.get_push_events_commits( company=company, from_date=date, to_date=date, date_period_type=DatePeriodType.DTD) schema = DataLake().staging.schemas.push_commits if events.empty: log.warning(f'No {company} events at {date}') return with GithubRest(token=Config().github_token) as rest: company_commits = get_company_repositories_events_commits( repositories_names=events[schema.repo_name].unique(), date=date, company=company, rest=rest) company_commits_df = pd.DataFrame(company_commits) DataLake().staging.save_private_push_events_commits( push_event_commits=company_commits_df, company_name=company, date=date)
def runClient(options): # log initialization log.message("Starting Outer Space Client", ige.version.versionString) log.debug("sys.path =", sys.path) log.debug("os.name =", os.name) log.debug("sys.platform =", sys.platform) log.debug("os.getcwd() =", os.getcwd()) log.debug("sys.frozen =", getattr(sys, "frozen", None)) # create required directories if not os.path.exists(options.configDir): os.makedirs(options.configDir) log.debug("options.configDir =", options.configDir) running = 1 first = True #while running: if not first: reload(osci) # parse configuration if first: import osci.gdata as gdata else: reload(gdata) gdata.config = Config( os.path.join(options.configDir, options.configFilename)) gdata.config.game.server = options.server setDefaults(gdata, options) language = gdata.config.client.language import gettext log.debug('OSCI', 'Installing translation for:', language) if language == 'en': log.debug('OSCI', 'English is native - installing null translations') tran = gettext.NullTranslations() else: try: tran = gettext.translation('OSPACE', resources.get('translations'), languages=[language]) except IOError: log.warning('OSCI', 'Cannot find catalog for', language) log.message('OSCI', 'Installing null translations') tran = gettext.NullTranslations() tran.install(unicode=1) #initialize pygame and prepare screen if (gdata.config.defaults.sound == "yes") or (gdata.config.defaults.music == "yes"): pygame.mixer.pre_init(44100, -16, 2, 4096) os.environ['SDL_VIDEO_ALLOW_SCREENSAVER'] = '1' os.environ['SDL_DEBUG'] = '1' pygame.init() flags = pygame.SWSURFACE DEFAULT_SCRN_SIZE = (800, 600) gdata.scrnSize = DEFAULT_SCRN_SIZE if gdata.config.display.resolution == "FULLSCREEN": gdata.scrnSize = (0, 0) flags |= pygame.FULLSCREEN elif gdata.config.display.resolution is not None: width, height = gdata.config.display.resolution.split('x') gdata.scrnSize = (int(width), int(height)) if gdata.config.display.depth == None: # guess best depth bestdepth = pygame.display.mode_ok(gdata.scrnSize, flags) else: bestdepth = int(gdata.config.display.depth) # initialize screen try: screen = pygame.display.set_mode(gdata.scrnSize, flags, bestdepth) # gdata.scrnSize is used everywhere to setup windows gdata.scrnSize = screen.get_size() except pygame.error: # for example if fullscreen is selected with resolution bigger than display # TODO: as of now, fullscreen has automatic resolution gdata.scrnSize = DEFAULT_SCRN_SIZE screen = pygame.display.set_mode(gdata.scrnSize, flags, bestdepth) gdata.screen = screen log.debug('OSCI', 'Driver:', pygame.display.get_driver()) log.debug('OSCI', 'Using depth:', bestdepth) log.debug('OSCI', 'Display info:', pygame.display.Info()) pygame.mouse.set_visible(1) pygame.display.set_caption(_('Outer Space %s') % ige.version.versionString) # set icon pygame.display.set_icon( pygame.image.load(resources.get('icon48.png')).convert_alpha()) # UI stuff if first: import pygameui as ui else: reload(ui) setSkinTheme(gdata, ui) app = ui.Application(update, theme=ui.SkinableTheme) app.background = defineBackground() app.draw(gdata.screen) app.windowSurfaceFlags = pygame.SWSURFACE | pygame.SRCALPHA gdata.app = app pygame.event.clear() # resources import osci.res osci.res.initialize() # load resources import osci.dialog dlg = osci.dialog.ProgressDlg(gdata.app) osci.res.loadResources(dlg) dlg.hide() osci.res.prepareUIIcons(ui.SkinableTheme.themeIcons) while running: if first: import osci.client, osci.handler from igeclient.IClient import IClientException else: reload(osci.client) reload(osci.handler) osci.client.initialize(gdata.config.game.server, osci.handler, options) # create initial dialogs if first: import osci.dialog else: reload(osci.dialog) gdata.savePassword = gdata.config.game.lastpasswordcrypted != None if options.login and options.password: gdata.config.game.lastlogin = options.login gdata.config.game.lastpassword = options.password gdata.config.game.lastpasswordcrypted = binascii.b2a_base64( options.password).strip() gdata.config.game.autologin = '******' gdata.savePassword = '******' loginDlg = osci.dialog.LoginDlg(gdata.app) updateDlg = osci.dialog.UpdateDlg(gdata.app) # event loop update() lastSave = time.clock() # set counter to -1 to trigger Update dialog (see "if" below) counter = -1 needsRefresh = False session = 1 first = False while running and session: try: counter += 1 if counter == 0: # display initial dialog in the very first cycle updateDlg.display(caller=loginDlg, options=options) # process as many events as possible before updating evt = pygame.event.wait() evts = pygame.event.get() evts.insert(0, evt) forceKeepAlive = False saveDB = False for evt in evts: if evt.type == pygame.QUIT: running = 0 break if evt.type == ( ui.USEREVENT) and evt.action == "localExit": session = False break if evt.type == pygame.ACTIVEEVENT: if evt.gain == 1 and evt.state == 6: # pygame desktop window focus event needsRefresh = True if evt.type == pygame.KEYUP and evt.key == pygame.K_F12: if not pygame.key.get_mods() & pygame.KMOD_CTRL: running = 0 break if evt.type == pygame.KEYUP and evt.key == pygame.K_F9: forceKeepAlive = True evt = gdata.app.processEvent(evt) if gdata.app.needsUpdate() or needsRefresh: needsRefresh = False update() # keep alive connection osci.client.keepAlive(forceKeepAlive) # save DB every 4 hours in case of a computer crash # using "counter" to limit calls to time.clock() to approximately every 10-15 minutes if counter > 5000: # set this to zero so we don't display Update dialog counter = 0 if time.clock() - lastSave > 14400: saveDB = True if saveDB: osci.client.saveDB() lastSave = time.clock() except IClientException, e: osci.client.reinitialize() gdata.app.setStatus(e.args[0]) loginDlg.display(message=e.args[0]) except Exception, e: log.warning('OSCI', 'Exception in event loop') if not isinstance(e, SystemExit) and not isinstance( e, KeyboardInterrupt): log.debug("Processing exception") # handle exception import traceback, StringIO fh = StringIO.StringIO() exctype, value, tb = sys.exc_info() funcs = [entry[2] for entry in traceback.extract_tb(tb)] faultID = "%06d-%03d" % ( hash("/".join(funcs)) % 1000000, traceback.extract_tb(tb)[-1][1] % 1000, ) del tb # high level info print >> fh, "Exception ID:", faultID print >> fh print >> fh, "%s: %s" % (exctype, value) print >> fh print >> fh, "--- EXCEPTION DATA ---" # dump exception traceback.print_exc(file=fh) excDlg = osci.dialog.ExceptionDlg(gdata.app) excDlg.display(faultID, fh.getvalue()) del excDlg # reference to the dialog holds app's intance fh.close() del fh else: break
def __get_blob_data_lakes( ) -> Tuple[BlobLandingArea, BlobStagingArea, BlobPublicArea]: return (BlobLandingArea(**Config().file_system.landing_props), BlobStagingArea(**Config().file_system.staging_props), BlobPublicArea(**Config().file_system.public_props))
def __get_local_data_lakes( ) -> Tuple[LocalLandingArea, LocalStagingArea, LocalPublicArea]: return (LocalLandingArea(**Config().file_system.landing_props), LocalStagingArea(**Config().file_system.staging_props), LocalPublicArea(**Config().file_system.public_props))
import ige.ospace.Rules as Rules Rules.init(options.configDir) from osci.StarMap import StarMap import resources import osci.res import pygame, pygame.ftfont, pygame.freetype import pygameui, pygameui.SkinableTheme from osci.config import Config import osci.gdata as gdata import osci.client import ige.version import gettext import hashlib gettext.NullTranslations().install(unicode=1) gdata.config = Config( os.path.join(options.configDir, options.configFilename)) gdata.config.game.server = 'localhost:9080' gdata.config.defaults.minfleetsymbolsize = 4 gdata.config.defaults.minplanetsymbolsize = 5 gdata.config.defaults.maxfleetsymbolsize = 0 gdata.config.defaults.maxplanetsymbolsize = 0 options.heartbeat = 60 osci.client.initialize('localhost:9080', dummyHandler, options) osci.client.login(options.game, 'admin', password) osci.client.cmdProxy.selectAdmin() osci.client.updateDatabase() pygame.init()
def test_config_singleton(): config1 = Config() config2 = Config() Config.tear_down() assert id(config1) == id(config2)
import logging from datetime import datetime import click logging.basicConfig(format='[%(asctime)s] [%(levelname)s] %(message)s', level=logging.DEBUG) log = logging.getLogger(__name__) if 'dbutils' in globals(): log.debug( 'Variable `dbutils` in memory. Try to setup config with `dbutils`') from osci.config import Config, FileSystemType config = Config(dbutils=dbutils) if 'spark' in globals(): log.debug('Variable `spark` in memory.') from osci.jobs.session import Session log.debug('Get spark session') Session(spark_session=spark) if Config().file_system_type == FileSystemType.blob: spark.conf.set( f'fs.azure.account.key.{Config().file_system.staging_props.get("storage_account_name")}.' f'blob.core.windows.net', Config().file_system.staging_props.get('storage_account_key')) from osci.actions import Action