def __endpoints__():
for params in endpoints:
print('\n>>> PARAMS __endpoints__() <<<')
print(params)
try:
enabler = db.query(EnablerImp).filter_by(name=params['enabler']).one()
except:
raise
else:
m_params = {'enabler_id': enabler.id}
for source in db.query(Source).all():
sourcename = source.name.lower()
if sourcename not in params:
continue
m_params['source_id'] = source.id
m_params['details'] = params[sourcename]
try:
metric = db.query(Metric).filter_by(source_id=source.id, enabler_id=enabler.id).one()
except NoResultFound:
db.add(Metric(**m_params))
else:
metric.details = m_params['details']
else:
db.commit()
def generate_data(self):
values = list()
values.append(
['Report date:',
datetime.now().strftime('%d %b %Y at %H:%m')])
# Add the date in which the database schema was created last time
values.append(['Data sources updated on:', self.getdate()])
values.append(['', ''])
header = ['Source']
for source in self.sources:
header.extend([source.name])
values.append(header)
units = ['Units']
for source in self.sources:
units.extend([source.units])
values.append(units)
values.append(['Enabler Implementation'])
values.append(['', ''])
for status in ('Incubated', 'Development', 'Support', 'Deprecated'):
values.append([status.upper(), ''])
for enabler in self.enablers:
raw = []
raw.extend([enabler.name])
if status != enabler.status:
continue
for source in self.sources:
try:
metric = db.query(Metric).filter_by(
source_id=source.id, enabler_id=enabler.id).one()
except NoResultFound:
raw.extend(['Not defined'])
continue
else:
if not len(metric.measurements):
raw.extend(['No data'])
continue
measurement = db.query(Measurement)\
.filter_by(metric_id=metric.id)\
.order_by(desc(Measurement.date))\
.first()
raw.extend([measurement.value])
values.append(raw)
values.append([''])
self.__save__(values=values)
def getdate():
"""
Get the creation date of the file enablers-dashboard.db
:return: the creation date of the file
"""
created_time = db.query(Admin).one().date.strftime('%d %b %Y at %H:%m')
return created_time
def __entities__():
for params in entities:
try:
entity = db.query(Entity).filter_by(shortname=params['shortname']).one()
except NoResultFound:
db.add(Entity(**params))
else:
setattr(entity, 'name', params['name'])
else:
db.commit()
def __sources__():
for params in sources:
try:
source = db.query(Source).filter_by(name=params['name']).one()
except NoResultFound:
db.add(Source(**params))
else:
for attr in ('content', 'url', 'api', 'units'):
if hasattr(source, attr) and attr in params:
setattr(source, attr, params[attr])
else:
db.commit()
def __enablers__():
for params in enablers:
print('\n>>> PARAMS <<<')
print(params)
try:
owner = db.query(Owner).filter_by(name=params['owner']).one()
except NoResultFound:
raise
else:
params['owner_id'] = owner.id
del params['owner']
try:
enabler = db.query(EnablerImp).filter_by(name=params['name']).one()
except NoResultFound:
db.add(EnablerImp(**params))
else:
for attr in ('chapter', 'type', 'status'):
if hasattr(enabler, attr) and attr in params:
setattr(enabler, attr, params[attr])
else:
db.commit()
def show_data():
print('\n>>> entities')
my_entities = db.query(Entity).all()
for entity in my_entities:
print(entity)
print('\n>>> owners')
my_owners = db.query(Owner).all()
for owner in my_owners:
print(owner)
print('\n>>> sources')
my_sources = db.query(Source).all()
for source in my_sources:
print(source)
print('\n>>> enablers')
my_enablers = db.query(EnablerImp).all()
for enabler in my_enablers:
print(enabler)
print('\n>>> metrics')
my_metrics = db.query(Metric).all()
for metric in my_metrics:
print(metric)
print('\n>>> measurements')
my_measurements = db.query(Measurement).all()
if len(my_measurements) == 0:
print("No measurements yet")
else:
for measurement in my_measurements:
print(measurement)
def __owners__():
for params in owners:
try:
entity = db.query(Entity).filter_by(shortname=params['entity']).one()
except NoResultFound:
raise
else:
params['entity_id'] = entity.id
del params['entity']
query = db.query(Owner).filter_by(name=params['name'])
try:
owner = query.one()
except NoResultFound:
db.add(Owner(**params))
db.commit()
else:
for attr in ('shortname', 'email'):
if hasattr(owner, attr) and attr in params:
setattr(owner, attr, params[attr])
else:
db.commit()
def obtain(self):
for source in db.query(Source):
logger.info(source.name)
# if source.name != 'Academy': continue
metrics = db.query(Metric).filter_by(source_id=source.id).all()
try:
op_source = eval('{}()'.format(source.name))
except Exception as e:
logger.error('source {} is not implemented'.format(
source.name))
logger.error(e)
continue
for metric in metrics:
try:
value = op_source.get_measurement(metric)
except NotDefined:
value = 'Not Defined'
except NotImplemented:
value = 'No Impl'
except InvalidConection:
value = 'No Connect'
except Exception as e:
logger.error(e)
value = 'No Access'
params = {
'metric_id': metric.id,
'date': datetime.now(),
'value': value.replace(',', '')
}
logger.debug(params)
measurement = Measurement(**params)
db.add(measurement)
else:
db.commit()
def __init__(self):
super(Docker, self).__init__()
self.source = db.query(Source).filter_by(name='Docker').one()
url = 'https://{}/u/fiware/'.format(self.source.url)
pattern = re.compile(r'"name":"[\w\-\.]*".*?"pull_count":\d*')
self.data = []
for n in range(10):
answer = requests.get(url, params={'page': n})
if not answer.ok:
continue
for match in re.finditer(pattern, answer.text):
self.data.append(json.loads('{' + match.group(0) + '}'))
def __init__(self):
super(Academy, self).__init__()
self.source = db.query(Source).filter_by(name='Academy').one()
self.ga = ga()
view = self.ga.search_view(self.source.url)
service = get_service('analyticsreporting')
body = {
'reportRequests': [{
'viewId':
view['profile_id'],
'dateRanges': [{
"startDate": "2015-09-01",
"endDate": date.today().strftime('%Y-%m-%d')
}],
'metrics': [{
'expression': 'ga:pageviews'
}],
'dimensions': [{
'name': 'ga:pageTitle'
}],
"dimensionFilterClauses": [{
"filters": [{
"dimensionName": "ga:pageTitle",
"operator": "BEGINS_WITH",
"expressions": "Course"
}]
}],
"orderBys": [{
"fieldName": "ga:pageviews",
"sortOrder": "DESCENDING"
}]
}],
}
try:
self.data = service.reports().batchGet(body=body).execute()
except:
raise
import re
import requests
import json
from dbase import db, Source
__author__ = 'Manuel Escriche'
source = db.query(Source).filter_by(name='Docker').one()
url = 'https://{}/u/fiware/'.format(source.url)
pattern = re.compile(r'"name":"[\w\-\.]*".*?"pull_count":\d*')
data = []
for n in range(10):
answer = requests.get(url, params={'page': n})
if not answer.ok:
continue
for match in re.finditer(pattern, answer.text):
data.append(json.loads('{' + match.group(0) + '}'))
for item in sorted(data, key=lambda x: x['name']):
print(item['name'], item['pull_count'])
def __init__(self):
super(Backlog, self).__init__()
self.source = db.query(Source).filter_by(name='Backlog').one()
self.scrm = ScrumServer(self.source.url)
def __init__(self):
super(Helpdesk, self).__init__()
self.source = db.query(Source).filter_by(name='Helpdesk').one()
self.scrm = ScrumServer(self.source.url)
def __init__(self):
super(GitHub, self).__init__()
self.source = db.query(Source).filter_by(name='GitHub').one()
self.gh = Github(login_or_token=GITHUB_TOKEN)
self.enabler_id = 0
def __init__(self):
super(Coverall, self).__init__()
self.source = db.query(Source).filter_by(name='Coverall').one()
self.url = 'https://{}/github'.format(self.source.url)
self.pattern = re.compile(
r'\<.*?id=\'repoShowPercentage\'\>(.*?)\<.*?\>', re.DOTALL)
import pprint
from datetime import date
from dbase import db, Source
from kernel.ganalytics import ga
from kernel.google import get_service
__author__ = 'Manuel Escriche'
source = db.query(Source).filter_by(name='Academy').one()
view = ga().search_view(source.url)
service = get_service('analyticsreporting')
body = {'reportRequests': [{'viewId': view['profile_id'],
'dateRanges': [{
"startDate": "2015-09-01",
"endDate": date.today().strftime('%Y-%m-%d')
}],
'metrics': [{'expression': 'ga:pageviews'}],
'dimensions': [{'name': 'ga:pageTitle'}],
"dimensionFilterClauses": [{
"filters": [{
"dimensionName": "ga:pageTitle",
"operator": "BEGINS_WITH",
"expressions": "Course"
}]
}],
# "orderBys":[{"fieldName": "ga:pageviews", "sortOrder": "DESCENDING"}]
}],
}
try:
data = service.reports().batchGet(body=body).execute()
def __init__(self):
super(Readthedocs, self).__init__()
self.source = db.query(Source).filter_by(name='Readthedocs').one()
self.ga = ga()
def __init__(self):
self.check_database()
self.enablers = db.query(EnablerImp)
self.sources = db.query(Source)
self.service = get_service('sheets')
from github import Github
from dbase import db, Source
from config.settings import GITHUB_TOKEN
__author__ = 'Manuel Escriche'
source = db.query(Source).filter_by(name='GitHub').one()
gh = Github(login_or_token=GITHUB_TOKEN)
for metric in source.metrics:
print('->', metric.enabler_imp.name)
if not metric.details:
print('{} metric not defined'.format(metric.enabler_imp.name))
continue
items = [metric.details] if isinstance(metric.details, str) else metric.details
for item in items:
user, project = item.split('/')
# print(user, project)
repo = gh.get_user(user).get_repo(project)
print('-->', repo.name, repo.html_url, 'updated at {} - pushed at {}'.format(repo.updated_at, repo.pushed_at))
releases = repo.get_releases()
for release in releases:
try:
print("\t->", release, ' published at {}'.format(release.raw_data['published_at']))
except:
continue
import re
import requests
import json
from dbase import db, Source
__author__ = 'Manuel Escriche'
source = db.query(Source).filter_by(name='Coverall').one()
keyword = 'Wirecloud/wirecloud'
url = 'https://{}/github/{}'.format(source.url, keyword)
# print(url)
pattern = re.compile(r'\<.*?id=\'repoShowPercentage\'\>(.*?)\<.*?\>', re.DOTALL)
answer = requests.get(url, params={'branch': 'HEAD'})
print(answer.url)
if not answer.ok:
raise Exception
match = re.search(pattern, answer.text)
if match:
print(match.group(1).strip())
else:
print('not match')
#print(answer.text)
import pprint
from datetime import date
from dbase import db, Source
from kernel.ganalytics import ga
from kernel.google import get_service
__author__ = 'Manuel Escriche'
source = db.query(Source).filter_by(name='Catalogue').one()
view = ga().search_view(source.url)
service = get_service('analyticsreporting')
body = {
'reportRequests': [{
'viewId':
view['profile_id'],
'dateRanges': [{
"startDate": "2015-09-01",
"endDate": date.today().strftime('%Y-%m-%d')
}],
'metrics': [{
'expression': 'ga:pageviews'
}],
'dimensions': [{
'name': 'ga:PagePath'
}],
"dimensionFilterClauses": [{
"filters": [{
"dimensionName": "ga:PagePath",
"operator": "BEGINS_WITH",
"expressions": "/enablers/"
}]
