class Connection:
''' Connects to league client and communicates with it '''
def __init__(self):
self.kwargs = None
self.url = None
self.session = FuturesSession()
def get_connection(self, settings):
''' Parses connection url and port from lockfile '''
raise NotImplementedError('Please implement this method')
def get(self, url, *args, **kwargs):
''' Wrapper around requests get method '''
return requests.get('{}{}'.format(self.url, url), *args, **kwargs, **self.kwargs)
def post(self, url, *args, **kwargs):
''' Wrapper around requests post method '''
return requests.post('{}{}'.format(self.url, url), *args, **kwargs, **self.kwargs)
def patch(self, url, *args, **kwargs):
''' Wrapper around requests patch method '''
return requests.patch('{}{}'.format(self.url, url), *args, **kwargs, **self.kwargs)
def put(self, url, *args, **kwargs):
''' Wrapper around requests put method '''
return requests.put('{}{}'.format(self.url, url), *args, **kwargs, **self.kwargs)
def delete(self, url, *args, **kwargs):
''' Wrapper around requests delete method '''
return requests.delete('{}{}'.format(self.url, url), *args, **kwargs, **self.kwargs)
def async_get(self, url, *args, **kwargs):
''' Wrapper around requests get method '''
return self.session.get('{}{}'.format(self.url, url), *args, **kwargs, **self.kwargs)
def async_post(self, url, *args, **kwargs):
''' Wrapper around requests post method '''
return self.session.post('{}{}'.format(self.url, url), *args, **kwargs, **self.kwargs)
def async_patch(self, url, *args, **kwargs):
''' Wrapper around requests patch method '''
return self.session.patch('{}{}'.format(self.url, url), *args, **kwargs, **self.kwargs)
def async_put(self, url, *args, **kwargs):
''' Wrapper around requests put method '''
return self.session.put('{}{}'.format(self.url, url), *args, **kwargs, **self.kwargs)
def async_delete(self, url, *args, **kwargs):
''' Wrapper around requests delete method '''
return self.session.delete('{}{}'.format(self.url, url), *args, **kwargs, **self.kwargs)
def get_api_call_future(api_call: ApiCall):
if api_call.method:
session = FuturesSession()
if api_call.method == 'GET':
return session.get(url=api_call.url)
elif api_call.method == 'POST':
return session.post(url=api_call.url, data=api_call.body)
elif api_call.method == 'PUT':
return session.put(url=api_call.url, data=api_call.body)
elif api_call.method == 'DELETE':
return session.delete(url=api_call.url)
else:
raise ValueError('Invalid method type: {}'.format(api_call.method))
else:
raise ValueError('No API method defined')
class SubjectAdmin:
def __init__(self, *, username, password, devilry_url):
self.devilry_url = devilry_url
self.rest_url = f'{devilry_url}/devilry_subjectadmin/rest'
self.period = None
self.session = FuturesSession(max_workers=24)
self.auth(username, password)
def auth(self, username, password):
self.creds = (username, password)
login_url = f'{self.devilry_url}/authenticate/login'
r = self.session.post(login_url,
{'username': username, 'password': password},
allow_redirects=False).result()
if not r.ok:
raise ConnectionError('Auth failed')
@staticmethod
def _json_cb(sess, resp):
resp.data = resp.json()
def get(self, url, *, cb=None, **kwargs):
cb = cb if not cb is None else self._json_cb
return self.session.get(f'{self.rest_url}/{url}', **kwargs,
background_callback=cb)
def post(self, url, **kwargs):
return self.session.post(f'{self.rest_url}/{url}', **kwargs,
auth=self.creds)
def put(self, url, **kwargs):
return self.session.put(f'{self.rest_url}/{url}', **kwargs,
auth=self.creds)
def delete(self, url, **kwargs):
return self.session.delete(f'{self.rest_url}/{url}', **kwargs,
auth=self.creds)
def periods(self):
courses = self.get('allwhereisadmin').result().data
periods = []
for course in courses:
for period in course['periods']:
periods.append({'course': course, 'period': period})
return periods
def set_period(self, period):
self.period = period
@needs_period
def create_assignment(self, *, short_name, long_name, first_deadline,
publishing_time, setupstudents_mode,
delivery_types=0, anonymous=False):
post_data = locals()
del post_data['self']
post_data['first_deadline'] = first_deadline.strftime('%F %T')
post_data['publishing_time'] = publishing_time.strftime('%F %T')
post_data['period_id'] = self.period['id']
return self.post('createnewassignment/', json=post_data,
background_callback=self._json_cb)
def set_hard_deadlines(self, assignment_id):
def task():
r = self.get(f'assignment/{assignment_id}').result()
assignment = r.data
assignment['deadline_handling'] = 1
r = self.put(f'assignment/{assignment_id}', json=assignment).result()
return self.session.executor.submit(task)
def set_points_assignment(self, assignment_id, min_points=0,
*, max_points, display_points=True):
def task():
r = self.get(f'assignment/{assignment_id}').result()
points2grade = 'raw-points' if display_points else 'passed-failed'
assignment = r.data
assignment['max_points'] = max_points
assignment['passing_grade_min_points'] = min_points
assignment['points_to_grade_mapper'] = points2grade
assignment['grading_system_plugin_id'] = \
'devilry_gradingsystemplugin_points'
r = self.put(f'assignment/{assignment_id}', json=assignment).result()
return self.session.executor.submit(task)
def examiner_stats(self, assignment_id):
return self.get(f'examinerstats/{assignment_id}')
def set_examiner(self, student, examiner, assignment):
return self.post(f'group/{assignment}/',
json={'candidates': [{'user': {'id': student}}],
'examiners': [{'user': examiner['user']}],
'is_open': True})
@lru_cache(maxsize=64)
def find_person(self, username):
r = self.session.get(f'{self.devilry_url}/devilry_usersearch/search'
f'?query={username}').result()
if not r.ok:
warn(f'Search could not be completed: {r.text}\n{r.reason}')
return
for user in r.json():
if user['username'] == username:
return user
def set_tags(self, assignment):
groups = self.get(f'group/{assignment}/').result().data
students = self.get(f'relatedstudent_assignment_ro/{assignment}/')\
.result().data
def get_tags(student):
for st in students:
if student == st['user']['id']:
return [{'tag': t} for t in st['tags'].split(',')]
futures = []
for group in groups:
# TODO: Copy ALL tags
f = self.put(f'group/{assignment}/',
json={'id': group['id'],
'candidates': [group['candidates'][0]],
'examiners': group['examiners'],
'is_open': True,
'tags': get_tags(group['candidates'][0]
['user']['id'])})
futures.append(f)
return futures
def get_group(self, username, assignment):
def cb(sess, resp):
for group in resp.json():
if group['candidates'][0]['user']['username'] == username:
resp.data = group
return
return self.get(f'group/{assignment}/?query={username}', cb=cb)
def update_examiner(self, group, examiner, assignment):
examiners = [] if examiner is None else [{'user': examiner['user']}]
return self.put(f'group/{assignment}/',
json={'id': group['id'],
'candidates': [group['candidates'][0]],
'examiners': examiners,
'is_open': True,
'tags': group['tags']})
def remove_students(self, students, assignment):
def remove(student):
r = self.get(f'group/{assignment}/?query={student}').result()
for group in r.data:
if group['candidates'][0]['user']['username'] == student:
break
else:
return
return self.delete(f'group/{assignment}/',
json={'id': group['id']}).result()
return [self.session.executor.submit(remove, student)
for student in students]
def remove_students_by_tag(self, tag, assignment):
r = self.get(f'group/{assignment}/?query={tag}').result()
futures = []
for group in r.data:
if tag in map(lambda x: x['tag'], group['tags']):
futures.append(self.delete(f'group/{assignment}/',
json={'id': group['id']}))
return futures
def add_students(self, students, assignment):
async def add(student):
r = await asyncio.wrap_future(
self.get(f"relatedstudent/{self.period['id']}?query={student}"))
if not r.ok:
warn(f'Student {student} could not be found:\n{r.text}')
for stud in r.json():
if stud['user']['username'] == student:
break
stud_id = stud['user']['id']
r = await asyncio.wrap_future(self.post(f'group/{assignment}/',
json={'candidates': [{'user': {'id': stud_id}}],
'is_open': True}))
if not r.ok:
warn(f'Student {student} could not be added:\n{r.text}')
loop = asyncio.get_event_loop()
g = asyncio.gather(*[add(student) for student in students])
loop.run_until_complete(g)
def setup_examiners_by_tags(self, assignment):
r = self.get(f"relatedexaminer/{self.period['id']}").result()
emap = {exr['tags']: exr['user']['id'] for exr in r.data}
r = self.get(f'group/{assignment}/').result()
futures = []
for group in r.data:
for tag in group['tags']:
t = tag['tag']
if not t in emap:
continue
futures.append(self.update_examiner(group,
{'user': {'id': emap[t]}},
assignment))
return futures
def close_groups_without_deliveries(self, assignment):
r = self.get(f'group/{assignment}/').result()
futures = []
for group in r.data:
if group['num_deliveries'] == 0:
futures.append(self.put(f'group/{assignment}/',
json={'id': group['id'],
'is_open': False,
'candidates': group['candidates'],
'examiners': group['examiners'],
'tags': group['tags']}))
return futures
def set_deadline_text(self, assignment, text):
dls = self.get(f'deadlinesbulk/{assignment}').result().data
futures = []
for dl in dls:
if dl['text'] is None:
futures.append(
self.put(f"deadlinesbulk/{assignment}/{dl['bulkdeadline_id']}",
json={'text': text,
'deadline': dl['deadline']}))
return futures
def remove_examiner_no_delivery(self, assignment):
r = self.get(f'group/{assignment}/').result()
futures = []
for group in r.data:
if not group['num_deliveries']:
futures.append(self.update_examiner(group, None,
assignment))
@needs_period
def points(self):
import pandas as pd
ov = {}
r = self.get(f"detailedperiodoverview/{self.period['id']}")
data = r.result().data
assignments = {a['id']: a['short_name'] for a in data['assignments']}
for student in r.result().data['relatedstudents']:
name = student['user']['username']
stdict = {}
for assignment in student['groups_by_assignment']:
a_name = assignments[assignment['assignmentid']]
if assignment['grouplist'] and assignment['grouplist'][0]['feedback']:
stdict[a_name] = assignment['grouplist'][0]['feedback']['points']
tag = student['relatedstudent']['tags'].split(',')[-1]
stdict['group'] = tag.replace('gruppe', '') if 'gruppe' in tag else None
ov[name] = stdict
df = pd.DataFrame(ov).T
df.set_index([df.index, 'group'], inplace=True)
return df
class CayenneApiClient:
def __init__(self, host):
self.host = host
self.auth = None
self.session = FuturesSession(executor=ThreadPoolExecutor(
max_workers=1))
def sendRequest(self, method, uri, body=None):
if self.session is not None:
headers = {}
request_url = self.host + uri
future = None
self.session.headers['Content-Type'] = 'application/json'
self.session.headers['Accept'] = 'application/json'
if self.auth is not None:
self.session.headers['Authorization'] = self.auth
try:
if method == 'GET':
future = self.session.get(request_url)
if method == 'POST':
future = self.session.post(request_url, data=body)
if method == 'PUT':
future = self.session.put(request_url, data=body)
if method == 'DELETE':
future = self.session.delete(request_url)
except Exception as ex:
error('sendRequest exception: ' + str(ex))
return None
try:
response = future.result()
except:
return None
return response
exception("No data received")
def getMessageBody(self, inviteCode):
body = {'id': inviteCode}
hardware = Hardware()
if hardware.Serial and hardware.isRaspberryPi():
body['type'] = 'rpi'
body['hardware_id'] = hardware.Serial
else:
hardware_id = hardware.getMac()
if hardware_id:
body['type'] = 'mac'
body['hardware_id'] = hardware_id
try:
system_data = []
cayennemqtt.DataChannel.add(system_data,
cayennemqtt.SYS_HARDWARE_MAKE,
value=hardware.getManufacturer(),
type='string',
unit='utf8')
cayennemqtt.DataChannel.add(system_data,
cayennemqtt.SYS_HARDWARE_MODEL,
value=hardware.getModel(),
type='string',
unit='utf8')
config = Config(APP_SETTINGS)
cayennemqtt.DataChannel.add(system_data,
cayennemqtt.AGENT_VERSION,
value=config.get(
'Agent', 'Version', __version__))
system_info = SystemInfo()
capacity_data = system_info.getMemoryInfo((cayennemqtt.CAPACITY, ))
capacity_data += system_info.getDiskInfo((cayennemqtt.CAPACITY, ))
for item in capacity_data:
system_data.append(item)
body['properties'] = {}
# body['properties']['pinmap'] = NativeGPIO().MAPPING
if system_data:
body['properties']['sysinfo'] = system_data
except:
exception('Error getting system info')
return json.dumps(body)
def authenticate(self, inviteCode):
body = self.getMessageBody(inviteCode)
url = '/things/key/authenticate'
return self.sendRequest('POST', url, body)
def activate(self, inviteCode):
body = self.getMessageBody(inviteCode)
url = '/things/key/activate'
return self.sendRequest('POST', url, body)
def getCredentials(self, content):
if content is None:
return None
body = content.decode("utf-8")
if body is None or body is "":
return None
return json.loads(body)
def loginDevice(self, inviteCode):
response = self.activate(inviteCode)
if response and response.status_code == 200:
return self.getCredentials(response.content)
return None
def getDataTypes(self):
url = '/ui/datatypes'
return self.sendRequest('GET', url)
class LogClient:
local_server = None
def __init__(self, url: str = None, max_workers=None):
if url.startswith("file://"):
self.local_server = LoggingServer(data_dir=url[6:])
elif os.path.isabs(url):
self.local_server = LoggingServer(data_dir=url)
elif url.startswith('http://'):
self.url = url
self.ping_url = os.path.join(url, "ping")
else:
# todo: add https://, and s3://
raise TypeError(
'log url need to begin with `/`, `file://` or `http://`.')
if max_workers:
self.session = FuturesSession(
ThreadPoolExecutor(max_workers=max_workers))
else:
self.session = FuturesSession()
def _get(self, key, dtype):
if self.local_server:
return self.local_server.load(key, dtype)
else:
json = LoadEntry(key, dtype)._asdict()
# note: reading stuff from the server is always synchronous via the result call.
res = self.session.get(self.url, json=json).result()
result = deserialize(res.text)
return result
def _post(self, key, data, dtype, options: LogOptions = None):
if self.local_server:
self.local_server.log(key, data, dtype, options)
else:
# todo: make the json serialization more robust. Not priority b/c this' client-side.
json = LogEntry(key, serialize(data), dtype, options)._asdict()
self.session.post(self.url, json=json)
def _delete(self, key):
if self.local_server:
self.local_server.remove(key)
else:
# todo: make the json serialization more robust. Not priority b/c this' client-side.
json = RemoveEntry(key)._asdict()
self.session.delete(self.url, json=json)
def ping(self, exp_key, status, _duplex=True, burn=True):
# todo: add configuration for early termination
if self.local_server:
signals = self.local_server.ping(exp_key, status)
return deserialize(signals) if _duplex else None
else:
# todo: make the json serialization more robust. Not priority b/c this' client-side.
ping_data = PingData(exp_key, status, burn=burn)._asdict()
req = self.session.post(self.ping_url, json=ping_data)
if _duplex:
response = req.result()
# note: I wonder if we should raise if the response is non-ok.
return deserialize(response.text) if response.ok else None
# send signals to the worker
def send_signal(self, exp_key, signal=None):
options = LogOptions(overwrite=True)
channel = os.path.join(exp_key, "__signal.pkl")
self._post(channel, signal, dtype="log", options=options)
# Reads binary data
def read(self, key):
return self._get(key, dtype="read")
# Reads binary data
def read_pkl(self, key):
return self._get(key, dtype="read_pkl")
def read_np(self, key):
return self._get(key, dtype="read_np")
# appends data
def log(self, key, data, **options):
self._post(key, data, dtype="log", options=LogOptions(**options))
# appends text
def log_text(self, key, text):
self._post(key, text, dtype="text")
# sends out images
def send_image(self, key, data):
assert data.dtype in ALLOWED_TYPES, "image data must be one of {}".format(
ALLOWED_TYPES)
self._post(key, data, dtype="image")
# appends text
def log_buffer(self, key, buf):
self._post(key, buf, dtype="byte")