def __endpoints__(): for params in endpoints: print('\n>>> PARAMS __endpoints__() <<<') print(params) try: enabler = db.query(EnablerImp).filter_by(name=params['enabler']).one() except: raise else: m_params = {'enabler_id': enabler.id} for source in db.query(Source).all(): sourcename = source.name.lower() if sourcename not in params: continue m_params['source_id'] = source.id m_params['details'] = params[sourcename] try: metric = db.query(Metric).filter_by(source_id=source.id, enabler_id=enabler.id).one() except NoResultFound: db.add(Metric(**m_params)) else: metric.details = m_params['details'] else: db.commit()
def generate_data(self): values = list() values.append( ['Report date:', datetime.now().strftime('%d %b %Y at %H:%m')]) # Add the date in which the database schema was created last time values.append(['Data sources updated on:', self.getdate()]) values.append(['', '']) header = ['Source'] for source in self.sources: header.extend([source.name]) values.append(header) units = ['Units'] for source in self.sources: units.extend([source.units]) values.append(units) values.append(['Enabler Implementation']) values.append(['', '']) for status in ('Incubated', 'Development', 'Support', 'Deprecated'): values.append([status.upper(), '']) for enabler in self.enablers: raw = [] raw.extend([enabler.name]) if status != enabler.status: continue for source in self.sources: try: metric = db.query(Metric).filter_by( source_id=source.id, enabler_id=enabler.id).one() except NoResultFound: raw.extend(['Not defined']) continue else: if not len(metric.measurements): raw.extend(['No data']) continue measurement = db.query(Measurement)\ .filter_by(metric_id=metric.id)\ .order_by(desc(Measurement.date))\ .first() raw.extend([measurement.value]) values.append(raw) values.append(['']) self.__save__(values=values)
def getdate(): """ Get the creation date of the file enablers-dashboard.db :return: the creation date of the file """ created_time = db.query(Admin).one().date.strftime('%d %b %Y at %H:%m') return created_time
def __entities__(): for params in entities: try: entity = db.query(Entity).filter_by(shortname=params['shortname']).one() except NoResultFound: db.add(Entity(**params)) else: setattr(entity, 'name', params['name']) else: db.commit()
def __sources__(): for params in sources: try: source = db.query(Source).filter_by(name=params['name']).one() except NoResultFound: db.add(Source(**params)) else: for attr in ('content', 'url', 'api', 'units'): if hasattr(source, attr) and attr in params: setattr(source, attr, params[attr]) else: db.commit()
def __enablers__(): for params in enablers: print('\n>>> PARAMS <<<') print(params) try: owner = db.query(Owner).filter_by(name=params['owner']).one() except NoResultFound: raise else: params['owner_id'] = owner.id del params['owner'] try: enabler = db.query(EnablerImp).filter_by(name=params['name']).one() except NoResultFound: db.add(EnablerImp(**params)) else: for attr in ('chapter', 'type', 'status'): if hasattr(enabler, attr) and attr in params: setattr(enabler, attr, params[attr]) else: db.commit()
def show_data(): print('\n>>> entities') my_entities = db.query(Entity).all() for entity in my_entities: print(entity) print('\n>>> owners') my_owners = db.query(Owner).all() for owner in my_owners: print(owner) print('\n>>> sources') my_sources = db.query(Source).all() for source in my_sources: print(source) print('\n>>> enablers') my_enablers = db.query(EnablerImp).all() for enabler in my_enablers: print(enabler) print('\n>>> metrics') my_metrics = db.query(Metric).all() for metric in my_metrics: print(metric) print('\n>>> measurements') my_measurements = db.query(Measurement).all() if len(my_measurements) == 0: print("No measurements yet") else: for measurement in my_measurements: print(measurement)
def __owners__(): for params in owners: try: entity = db.query(Entity).filter_by(shortname=params['entity']).one() except NoResultFound: raise else: params['entity_id'] = entity.id del params['entity'] query = db.query(Owner).filter_by(name=params['name']) try: owner = query.one() except NoResultFound: db.add(Owner(**params)) db.commit() else: for attr in ('shortname', 'email'): if hasattr(owner, attr) and attr in params: setattr(owner, attr, params[attr]) else: db.commit()
def obtain(self): for source in db.query(Source): logger.info(source.name) # if source.name != 'Academy': continue metrics = db.query(Metric).filter_by(source_id=source.id).all() try: op_source = eval('{}()'.format(source.name)) except Exception as e: logger.error('source {} is not implemented'.format( source.name)) logger.error(e) continue for metric in metrics: try: value = op_source.get_measurement(metric) except NotDefined: value = 'Not Defined' except NotImplemented: value = 'No Impl' except InvalidConection: value = 'No Connect' except Exception as e: logger.error(e) value = 'No Access' params = { 'metric_id': metric.id, 'date': datetime.now(), 'value': value.replace(',', '') } logger.debug(params) measurement = Measurement(**params) db.add(measurement) else: db.commit()
def __init__(self): super(Docker, self).__init__() self.source = db.query(Source).filter_by(name='Docker').one() url = 'https://{}/u/fiware/'.format(self.source.url) pattern = re.compile(r'"name":"[\w\-\.]*".*?"pull_count":\d*') self.data = [] for n in range(10): answer = requests.get(url, params={'page': n}) if not answer.ok: continue for match in re.finditer(pattern, answer.text): self.data.append(json.loads('{' + match.group(0) + '}'))
def __init__(self): super(Academy, self).__init__() self.source = db.query(Source).filter_by(name='Academy').one() self.ga = ga() view = self.ga.search_view(self.source.url) service = get_service('analyticsreporting') body = { 'reportRequests': [{ 'viewId': view['profile_id'], 'dateRanges': [{ "startDate": "2015-09-01", "endDate": date.today().strftime('%Y-%m-%d') }], 'metrics': [{ 'expression': 'ga:pageviews' }], 'dimensions': [{ 'name': 'ga:pageTitle' }], "dimensionFilterClauses": [{ "filters": [{ "dimensionName": "ga:pageTitle", "operator": "BEGINS_WITH", "expressions": "Course" }] }], "orderBys": [{ "fieldName": "ga:pageviews", "sortOrder": "DESCENDING" }] }], } try: self.data = service.reports().batchGet(body=body).execute() except: raise
import re import requests import json from dbase import db, Source __author__ = 'Manuel Escriche' source = db.query(Source).filter_by(name='Docker').one() url = 'https://{}/u/fiware/'.format(source.url) pattern = re.compile(r'"name":"[\w\-\.]*".*?"pull_count":\d*') data = [] for n in range(10): answer = requests.get(url, params={'page': n}) if not answer.ok: continue for match in re.finditer(pattern, answer.text): data.append(json.loads('{' + match.group(0) + '}')) for item in sorted(data, key=lambda x: x['name']): print(item['name'], item['pull_count'])
def __init__(self): super(Backlog, self).__init__() self.source = db.query(Source).filter_by(name='Backlog').one() self.scrm = ScrumServer(self.source.url)
def __init__(self): super(Helpdesk, self).__init__() self.source = db.query(Source).filter_by(name='Helpdesk').one() self.scrm = ScrumServer(self.source.url)
def __init__(self): super(GitHub, self).__init__() self.source = db.query(Source).filter_by(name='GitHub').one() self.gh = Github(login_or_token=GITHUB_TOKEN) self.enabler_id = 0
def __init__(self): super(Coverall, self).__init__() self.source = db.query(Source).filter_by(name='Coverall').one() self.url = 'https://{}/github'.format(self.source.url) self.pattern = re.compile( r'\<.*?id=\'repoShowPercentage\'\>(.*?)\<.*?\>', re.DOTALL)
import pprint from datetime import date from dbase import db, Source from kernel.ganalytics import ga from kernel.google import get_service __author__ = 'Manuel Escriche' source = db.query(Source).filter_by(name='Academy').one() view = ga().search_view(source.url) service = get_service('analyticsreporting') body = {'reportRequests': [{'viewId': view['profile_id'], 'dateRanges': [{ "startDate": "2015-09-01", "endDate": date.today().strftime('%Y-%m-%d') }], 'metrics': [{'expression': 'ga:pageviews'}], 'dimensions': [{'name': 'ga:pageTitle'}], "dimensionFilterClauses": [{ "filters": [{ "dimensionName": "ga:pageTitle", "operator": "BEGINS_WITH", "expressions": "Course" }] }], # "orderBys":[{"fieldName": "ga:pageviews", "sortOrder": "DESCENDING"}] }], } try: data = service.reports().batchGet(body=body).execute()
def __init__(self): super(Readthedocs, self).__init__() self.source = db.query(Source).filter_by(name='Readthedocs').one() self.ga = ga()
def __init__(self): self.check_database() self.enablers = db.query(EnablerImp) self.sources = db.query(Source) self.service = get_service('sheets')
from github import Github from dbase import db, Source from config.settings import GITHUB_TOKEN __author__ = 'Manuel Escriche' source = db.query(Source).filter_by(name='GitHub').one() gh = Github(login_or_token=GITHUB_TOKEN) for metric in source.metrics: print('->', metric.enabler_imp.name) if not metric.details: print('{} metric not defined'.format(metric.enabler_imp.name)) continue items = [metric.details] if isinstance(metric.details, str) else metric.details for item in items: user, project = item.split('/') # print(user, project) repo = gh.get_user(user).get_repo(project) print('-->', repo.name, repo.html_url, 'updated at {} - pushed at {}'.format(repo.updated_at, repo.pushed_at)) releases = repo.get_releases() for release in releases: try: print("\t->", release, ' published at {}'.format(release.raw_data['published_at'])) except: continue
import re import requests import json from dbase import db, Source __author__ = 'Manuel Escriche' source = db.query(Source).filter_by(name='Coverall').one() keyword = 'Wirecloud/wirecloud' url = 'https://{}/github/{}'.format(source.url, keyword) # print(url) pattern = re.compile(r'\<.*?id=\'repoShowPercentage\'\>(.*?)\<.*?\>', re.DOTALL) answer = requests.get(url, params={'branch': 'HEAD'}) print(answer.url) if not answer.ok: raise Exception match = re.search(pattern, answer.text) if match: print(match.group(1).strip()) else: print('not match') #print(answer.text)
import pprint from datetime import date from dbase import db, Source from kernel.ganalytics import ga from kernel.google import get_service __author__ = 'Manuel Escriche' source = db.query(Source).filter_by(name='Catalogue').one() view = ga().search_view(source.url) service = get_service('analyticsreporting') body = { 'reportRequests': [{ 'viewId': view['profile_id'], 'dateRanges': [{ "startDate": "2015-09-01", "endDate": date.today().strftime('%Y-%m-%d') }], 'metrics': [{ 'expression': 'ga:pageviews' }], 'dimensions': [{ 'name': 'ga:PagePath' }], "dimensionFilterClauses": [{ "filters": [{ "dimensionName": "ga:PagePath", "operator": "BEGINS_WITH", "expressions": "/enablers/" }]