def get_by_server_type():
with session_scope() as session:
records = session.query(VisitedLinksModel.server_type,
func.count(
VisitedLinksModel.server_type)).group_by(
VisitedLinksModel.server_type).all()
return server_type_simplifier(records)
def update_repository(update):
with session_scope() as session:
if 'owner_id' in update['updates']:
github_user_id = update['updates']['owner_id']
user = session.query(User).filter(User.github_id == github_user_id).first()
if user is not None:
update['updates']['owner_id'] = user.id
else:
return
session.query(Repository).filter(Repository.id == update['id']).update(update['updates'])
def add_or_update_contribution(contribution):
with session_scope() as session:
q = session.query(Contribution).filter(Contribution.contributor == contribution.contributor and
Contribution.repository == contribution.repository).first()
if q is not None:
q.additions = contribution.additions
q.deletions = contribution.deletions
session.add(q)
else:
session.add(contribution)
def find_or_create_user_by_email(email):
with session_scope() as session:
e = session.query(Email).filter(Email.email == email).first()
if e is None:
user = User(emails=[Email(email=email)])
session.add(user)
session.flush()
id = user.id
else:
id = e.user_id
return id
def add_or_update_user(github_id, login, gravatar_id, email=None,
name=None, followers=None, following=None):
with session_scope() as session:
ret = session.query(User).filter(User.github_id == github_id).first()
if ret:
ret.github_id = github_id
ret.login = login
ret.gravatar = gravatar_id
if email is not None:
ret.emails.append(email)
ret.name = name
ret.followers = followers
ret.following = following
else:
if email is not None:
u = User(github_id=github_id, login=login, gravatar=gravatar_id, emails=[email],
name=name, followers=followers, following=following)
else:
u = User(github_id=github_id, login=login, gravatar=gravatar_id,
name=name, followers=followers, following=following)
session.add(u)
def get_by_server_type_time(year=0,
month=0,
day=0,
hour=0,
minute=0,
second=0):
now = datetime.now()
start = now - relativedelta(years=year,
months=month,
days=day,
hours=hour,
minutes=minute,
seconds=second)
with session_scope() as session:
records = session.query(
VisitedLinksModel.server_type,
func.count(VisitedLinksModel.server_type)).filter(
VisitedLinksModel.saved > start).filter(
VisitedLinksModel.saved < now).group_by(
VisitedLinksModel.server_type).all()
return server_type_simplifier(records)
def agent(args):
"""Run a long running polling agent.
Periodically updates the database with timeseries active stat data.
:param args: argparse args
"""
conf = config.load(args.config_file)
nova = get_nova(conf)
database.get_engine(conf.get('db', {}))
detailed = True
# Special all tenant.
all_tenant_id = '0' * 32
# Init last poll time to 0 to trigger first poll
last_polled = 0
while True:
if time.time() - last_polled > conf.get('polling_interval', 120):
with database.session_scope() as session:
start, end = utils.get_date_interval()
# Grab usage results
usages = nova.usage.list(start, end, detailed=detailed)
r = UsageReport(usages)
# Get datetime for time value
now = utils.get_now()
# Iterate over all tenants
for tenant_usage in usages:
# Ensure tenant is in tenant table
tenant_id = tenant_usage.tenant_id
tenant = Tenant.get_or_create(session, tenant_id)
session.commit()
# Get tenant stats and add to session
v, m, s = r.active_stats(tenant_id=tenant_id)
session.add(ActiveVCPUS(
value=v, time=now, tenant_id=tenant.id
))
session.add(ActiveMemoryMB(
value=m, time=now, tenant_id=tenant.id
))
session.add(ActiveLocalStorageGB(
value=s, time=now, tenant_id=tenant.id
))
# Save all tenant stats
v, m, s = r.active_stats()
print ("Active vcpus", v)
print ("Active memory MB", m)
print ("Active storage GB", s)
all_tenant = Tenant.get_or_create(session, all_tenant_id)
session.commit()
session.add(ActiveVCPUS(
value=v, time=now, tenant_id=all_tenant.id
))
session.add(ActiveMemoryMB(
value=m, time=now, tenant_id=all_tenant.id
))
session.add(ActiveLocalStorageGB(
value=s, time=now, tenant_id=all_tenant.id
))
last_polled = time.time()
print ("Updating polling interval")
time.sleep(1)
exit()
def add_links(self, links):
with session_scope() as session:
session.bulk_save_objects(links)
def size(self):
size = 0
with session_scope() as session:
size = session.query(VisitedLinksModel).count()
return size
def get_repository(id):
with session_scope() as session:
repo = session.query(Repository).filter(Repository.id == id).first()
session.expunge_all()
assert(repo is not None)
return repo
def add_repository(id, name):
with session_scope() as session:
session.add(Repository(id=id, name=name))
def read_queued_tasks():
with session_scope() as session:
q = session.query(UpdateRequest)
for r in q:
git_tasks.generate_repository_stats.delay(r.repository_id)
session.delete(r)
def get_last_repository_id():
ret = None
with session_scope() as session:
(ret, ) = session.query(func.max(Repository.id)).first()
return ret
def crawl(self, queue_links):
with session_scope() as session:
session.add_all(queue_links)
def return_records(self):
with session_scope() as session:
records = session.query(NotVisitedLinksModel).filter(
NotVisitedLinksModel.taken == False).limit(20).all()
return records