def _on_first_grace(self, config):
should_ping_reviewer = False
assignee_comments = filter(lambda d: d['who'] == self.data['assignee'],
self.data['comments'])
author_comments = filter(lambda d: d['who'] == self.data['author'],
self.data['comments'])
if not assignee_comments:
# Reviewer hasn't commented at all!
should_ping_reviewer = True
else:
last_review = datetime_parse(assignee_comments[-1]['when'])
last_push = datetime_parse(self.data['last_push'])
if last_review < last_push:
# Reviewer hasn't looked at this since the last push
should_ping_reviewer = True
elif not author_comments:
# Author hasn't commented at all!
comment = self.api.rand_choice(config['pr_ping'])
self.api.post_comment(
comment.format(author=self.data['author']))
self.data['status'] = 'commented'
else:
# It could be waiting on the assignee or the author. Right now, we just poke them both.
self.api.post_comment(
self.api.rand_choice(config['pr_anon_ping']))
if should_ping_reviewer:
# Right now, we just ping the reviewer until he takes a look at this or assigns someone else
comment = self.api.rand_choice(config['review_ping'])
self.api.post_comment(
comment.format(reviewer=self.data['assignee']))
def test_contributors_update(self):
'''
Contributors list (cache) live only for an hour (by default). Once it's outdated,
the next call to `get_contributors` calls `fetch_contributors`, writes it to the store
and returns the list. Any calls within the next hour will return the existing contributors
without calling the API.
'''
class TestAPI(APIProvider):
fetched = False
def fetch_contributors(self):
self.fetched = True
return []
config = create_config()
api = TestAPI(config=config, payload={}, store=None)
self.assertFalse(api.fetched)
api.get_contributors()
# No store. This will always call the API.
self.assertTrue(api.fetched)
store = TestStore()
api = TestAPI(config=config, payload={}, store=store)
self.assertFalse(api.fetched)
now = datetime.now()
api.get_contributors()
data = store.get_object(CONTRIBUTORS_STORE_KEY)
updated_time = datetime_parse(data['last_update_time'])
# Store doesn't have contributors. It's been updated for the first time.
self.assertTrue(updated_time >= now)
self.assertTrue(api.fetched)
store = TestStore()
store.write_object(CONTRIBUTORS_STORE_KEY, {
'last_update_time': str(now),
'list': ['booya']
})
api = TestAPI(config=config, payload={}, store=store)
self.assertFalse(api.fetched)
api.get_contributors()
data = store.get_object(CONTRIBUTORS_STORE_KEY)
updated_time = datetime_parse(data['last_update_time'])
# Called within a cycle - no fetch occurs.
self.assertEqual(updated_time, now)
self.assertFalse(api.fetched)
store = TestStore()
store.write_object(CONTRIBUTORS_STORE_KEY, {
'last_update_time': str(now),
'list': ['booya']
})
api = TestAPI(config=config, payload={}, store=store)
self.assertFalse(api.fetched)
api.get_contributors(fetch=True)
# When `fetch` is enabled, API is called regardless.
self.assertTrue(api.fetched)
data = store.get_object(CONTRIBUTORS_STORE_KEY)
updated_time = datetime_parse(data['last_update_time'])
self.assertTrue(updated_time > now)
def get_fulfillable_course_runs_for_entitlement(entitlement, course_runs):
"""
Takes a list of course runs and returns only the course runs, sorted by start date, that:
1) Are currently running or in the future
2) A user can enroll in
3) A user can upgrade in
4) Are published
5) Are not enrolled in already for an active session
These are the only sessions that can be selected for an entitlement.
"""
enrollable_sessions = []
enrollments_for_user = CourseEnrollment.enrollments_for_user(
entitlement.user).filter(mode=entitlement.mode)
enrolled_sessions = frozenset(
[str(e.course_id) for e in enrollments_for_user])
# Only show published course runs that can still be enrolled and upgraded
now = datetime.datetime.now(UTC)
for course_run in course_runs:
# Only courses that have not ended will be displayed
run_start = course_run.get('start')
run_end = course_run.get('end')
is_running = run_start and (not run_end
or datetime_parse(run_end) > now)
# Only courses that can currently be enrolled in will be displayed
enrollment_start = course_run.get('enrollment_start')
enrollment_end = course_run.get('enrollment_end')
can_enroll = (
(not enrollment_start or datetime_parse(enrollment_start) < now)
and (not enrollment_end or datetime_parse(enrollment_end) > now)
and course_run.get('key') not in enrolled_sessions)
# Only upgrade-able courses will be displayed
can_upgrade = False
for seat in course_run.get('seats', []):
if seat.get('type') == entitlement.mode:
upgrade_deadline = seat.get('upgrade_deadline', None)
can_upgrade = not upgrade_deadline or (
datetime_parse(upgrade_deadline) > now)
break
# Only published courses will be displayed
is_published = course_run.get('status') == 'published'
if is_running and can_upgrade and can_enroll and is_published:
enrollable_sessions.append(course_run)
enrollable_sessions.sort(key=lambda session: session.get('start'))
return enrollable_sessions
def test_screens_after_earliest_datetime(self):
datetime = datetime_parse('2018-01-01T00:00:01+00:00')
video = AnnotatedScreenVideo(INSTITUTION_ID, ROOM_ID, CAMERA_ID,
datetime)
frame = next(iter(video))
self.assertEqual(
set([
('aspect_ratio', datetime_parse('2018-01-01T00:00:00+00:00')),
('no_from_no_until',
datetime_parse('2018-01-01T00:00:00+00:00')),
('early_from_no_until',
datetime_parse('2018-01-01T00:00:00+00:00')),
('equal_from_no_until',
datetime_parse('2018-01-01T00:00:00+00:00')),
('late_from_no_until',
datetime_parse('2018-01-01T00:00:00+00:00')),
('no_from_early_until',
datetime_parse('2018-01-01T00:00:00+00:00')),
('no_from_equal_until',
datetime_parse('2018-01-01T00:00:00+00:00')),
('no_from_late_until',
datetime_parse('2018-01-01T00:00:00+00:00')),
]),
set([(screen.screen_id, screen.datetime)
for screen in self.screen_detector.detect(frame)]),
)
def parse_bounding_dates(data):
if '_start_at' in data:
start_at = datetime_parse(data['_start_at'])
if '_end_at' in data:
end_at = datetime_parse(data['_end_at'])
else:
end_at = datetime.datetime.now(pytz.UTC).replace(microsecond=0)
else:
return (None, None)
return (start_at, end_at)
def parse_bounding_dates(data):
if '_start_at' in data:
start_at = datetime_parse(data['_start_at'])
if '_end_at' in data:
end_at = datetime_parse(data['_end_at'])
else:
end_at = datetime.datetime.now(pytz.UTC).replace(microsecond=0)
else:
return (None, None)
return (start_at, end_at)
def parse_post_expires(self, response):
meta = response.xpath('//meta[@name="robots"]')
if meta:
for robot_param in meta.attrib["content"].split(","):
if robot_param.startswith("unavailable_after"):
dt = robot_param.replace("unavailable_after: ", "")
return datetime_parse(dt)
def get_contributors(self, fetch=False):
'''
If `fetch` is disabled and if store exists, then this gets the contributors list from the
store. Otherwise, this calls the overriddable method, gets the list, writes to the store
(if it exists) and returns the list.
'''
now = datetime.now()
if self.store:
contributors = self.store.get_object(CONTRIBUTORS_STORE_KEY)
if contributors:
# Force fetch if last updated time is long back.
timestamp = contributors.get('last_update_time', str(now))
last_update = datetime_parse(timestamp)
if (now - last_update).seconds >= CONTRIBUTORS_UPDATE_INTERVAL_HOURS * 3600:
fetch = True
if contributors.get('list') and not fetch:
return contributors['list']
self.logger.info('Updating contributors list...')
contributors = {
'last_update_time': str(now),
'list': self.fetch_contributors()
}
if self.store:
self.store.write_object(CONTRIBUTORS_STORE_KEY, data=contributors)
return contributors['list']
def _clean_metadata(metadata):
meta = copy.deepcopy(DEFAULT_META)
meta.update(metadata or {}) # handles None metadata
meta['tags'] = set(meta.get('tags', [])) # tags should be a set
if 'added' in meta and isinstance(meta['added'], ustr):
meta['added'] = datetime_parse(meta['added'])
return meta
def sync_token(self):
'''
Prove your ownership over the integration to Github and request a token.
This token is used for API requests in the future. It has a lifetime,
and should be sync'ed again later.
'''
now = datetime.now(
self.next_token_sync.tzinfo) # timezone-aware version
if now < self.next_token_sync:
return
self.logger.debug('Getting auth token with JWT from PEM key...')
# https://developer.github.com/apps/building-github-apps/authentication-options-for-github-apps/
since_epoch = int(time.time())
auth_payload = {
'iat': since_epoch,
'exp': since_epoch + 600, # 10 mins expiration for JWT
'iss': self.config.integration_id,
}
auth = 'Bearer %s' % jwt.encode(auth_payload, self.config.pem_key,
'RS256')
resp = self._request('POST', self.installation_url, auth=auth)
self.token = resp.data[
'token'] # installation token (expires in 1 hour)
self.logger.debug('Token expires on %s', resp.data['expires_at'])
self.next_token_sync = datetime_parse(resp.data['expires_at'])
def test_no_screens_before_earliest_datetime(self):
datetime = datetime_parse('2017-12-31T23:59:59+00:00')
video = AnnotatedScreenVideo(INSTITUTION_ID, ROOM_ID, CAMERA_ID,
datetime)
frame = next(iter(video))
screens = set(self.screen_detector.detect(frame))
self.assertEqual(screens, set())
def score_document(self, doc):
"""
doc:
created_at
url
title
keywords
"""
max_id = None
last_created_date = doc.created_at
score = 0
while last_created_date >= doc.created_at:
results = self.api.get_keyword_tweets(doc.keywords,
max_id)['statuses']
print 'got {} tweets'.format(len(results))
print 'last date', last_created_date
for tweet in results:
score += self.compute_score(doc, tweet)
max_id = int(tweet['id'])
last_created_date = datetime_parse(tweet['created_at'])
if last_created_date < doc.created_at:
return score
if len(results) == 0:
return score
max_id -= 1
time.sleep(4) # rate limit 450 per 30 minutes
return score
def _prepare_for_update(self):
config = self.get_matched_subconfig()
if not config:
return
self.logger.info('Preparing to post weekly update...')
assignees = self.join_names(map(lambda n: '@' + n,
config['assignees']))
week_end = datetime_parse(self.data['post_date'])
week_start = str((week_end - timedelta(days=7)).date())
week_end = str(week_end.date())
newcomers = '\n'.join(map(lambda c: ' - @' + c,
self.data['newcomers']))
title = config['issue_title'].format(date=week_end)
body = TEMPLATE.format(assignees=assignees,
newcomers=newcomers,
start=week_start,
end=week_end,
owner=self.data['owner'],
repo=self.data['repo'])
owner, repo = config['repo'].split('/')
with Modifier(self.api, owner=owner, repo=repo):
self.api.create_issue(title=title,
body=body,
assignees=config.get('assignees', []),
labels=config.get('labels', []))
# Removing the data, so that next time a PR is opened, we'll start afresh
self.logger.info('Removing existing data...')
self.remove_object()
def datetime_interpretation(raw):
"""Converts human-readable datetime into normal datetime instance.
Args:
raw (str or unicode): source string with datetime data.
Returns:
datetime: result datetime instance.
None: if couldn't read value.
"""
if isinstance(raw, (unicode, str)) and len(raw):
if isinstance(raw, str):
try:
raw = unicode(raw, 'utf-8')
except UnicodeDecodeError:
return None
try:
for ru, en in RU_EN_MONTHS:
if ru in raw:
raw = raw.replace(ru, en)
break
return datetime_parse(raw)
except:
return None
else:
return None
def _check_pulls(self):
config = self.get_matched_subconfig()
if not config:
return
for number in self.pr_list['pulls']:
self.data = self.get_object(key=number)
self.old_data = deepcopy(self.data)
last_active = self.data.get('last_active')
if not last_active:
continue
last_active = datetime_parse(last_active)
now = datetime.now(last_active.tzinfo)
if (now - last_active).days <= config['grace_period_days']:
self.logger.debug('PR #%s is stil in grace period', number)
continue
self.logger.info(
"PR #%s has had its time. Something's gonna happen.", number)
self.data['last_active'] = str(now)
with Modifier(self.api, number=number):
self._handle_indiscipline_pr(config)
if self.old_data != self.data:
self.write_object(self.data, key=number)
def _update_data(self, data={}):
'''Update the data in this object.'''
# Store the changes to prevent this update from effecting it
pending_changes = self._changes or {}
try:
del self._changes
except:
pass
# Map custom fields into our custom fields object
try:
custom_field_data = data.pop('custom_fields')
except KeyError:
pass
else:
self.custom_fields = Custom_Fields(custom_field_data)
# Map all other dictionary data to object attributes
for key, value in data.iteritems():
lookup_key = self._field_type.get(key, key)
# if it's a datetime object, turn into proper DT object
if lookup_key == 'datetime' or lookup_key == 'date':
self.__dict__[key] = datetime_parse(value)
else:
# Check to see if there's cache data for this item.
# Will return an object if it's recognized as one.
self.__dict__[key] = self._redmine.check_cache(lookup_key, value)
#self.__dict__.update(data)
# Set the changes dict to track all changes from here on out
self._changes = pending_changes
def test_produces_single_frame(self):
datetime = datetime_parse('2018-01-01T00:00:00+00:00')
video = AnnotatedScreenVideo(INSTITUTION_ID, ROOM_ID, CAMERA_ID,
datetime)
frame_iterator = iter(video)
next(frame_iterator)
with self.assertRaises(StopIteration):
next(frame_iterator)
def test_aspect_ratio(self):
datetime = datetime_parse('2018-01-01T00:00:00+00:00')
video = AnnotatedScreenVideo(INSTITUTION_ID, ROOM_ID, CAMERA_ID,
datetime)
frame = next(iter(video))
self.assertIn(('aspect_ratio', 133, 100),
set([(screen.screen_id, screen.width, screen.height)
for screen in self.screen_detector.detect(frame)]))
def _isTokenExpired(self):
# static auth does not expire
if len(self.api_token_expires) == 0:
return False
expiration = datetime_parse(self.api_token_expires)
return expiration < datetime.datetime.now()
def convert_string_for_numerical_filter(value: str):
try:
return float(value)
except ValueError:
try:
return datetime_parse(value)
except ParserError:
raise (ValueError, "neither a float nor a datetime")
def __load_data(self):
"""
Loads in the aerodynamic data from a file.
:raises ValueError: If an unknown model was defined in the file.
"""
if "r" in self.mode:
# Load raw data from file.
raw_data = flex_load(file_path=self.path,
default_serializer=msgpack,
default_is_gzipped=True)
# Validate data, raise error if the data is invalid.
valid, error = self.validate_data(raw_data)
if not valid:
raise ValueError("Invalid aerodynamic file.\n" + yaml.dump(error))
# Read in parameters.
self.case = raw_data['case']
self.description = raw_data['description']
self.lref = raw_data['lref']
self.sref = raw_data['sref']
self.latref = raw_data['latref']
self.mrc = np.array(raw_data['mrc']).reshape((3, 1))
# Load in the created datetime.
# The datetime should be stored as a string in the data file, but
# some serializer, like the yaml loader, will automatically parse it into a
# datetime object.
if issubclass(type(raw_data['created']), datetime):
self.created = raw_data['created']
else:
self.created = datetime_parse(raw_data['created'])
# Loop through the coefficient entries an create an instance of AeroMethodBase for each.
for c_data in raw_data['coefficients']:
# Get the class for the specified model.
model_class = aero_models_registry.get(c_data['model'])
# Check whether a class for the model was found. If not raise an exception.
if model_class is None:
raise ValueError("Aerodynamic model '{}' unknown.".format(c_data['model']))
else:
# Validate the properties dictionary.
result = model_class.validate_data(c_data['properties'])
if result is not None:
raise ValueError("Invalid aerodynamic file.\n" + result)
# If a class was found, create an instance of it and pass the parameters to the
# constructor and store the resulting object in the coefficients table.
coefficient_model = model_class(**c_data['properties'])
# Check whether are parameters are valid.
# TODO: Finish writing this.
self.add_coefficient(c_data['name'], coefficient_model)
def get_download_metadata(self):
"""
Returns basic metadata about the current CAL-ACCESS snapshot,
like its size and the last time it was updated while stopping
short of actually downloading it.
"""
request = requests.head(self.url)
return {
'content-length': int(request.headers['content-length']),
'last-modified': datetime_parse(request.headers['last-modified'])
}
def get_download_metadata(self):
"""
Returns basic metadata about the current CAL-ACCESS snapshot,
like its size and the last time it was updated while stopping
short of actually downloading it.
"""
request = requests.head(self.url)
return {
'content-length': int(request.headers['content-length']),
'last-modified': datetime_parse(request.headers['last-modified'])
}
def fetch_shares(self, limit=1000, **kwargs):
"""
Retrieve and save all shares of post
"""
from facebook_api.models import MASTER_DATABASE # here, becouse cycling import
ids = []
response = api_call('%s/sharedposts' % self.graph_id, **kwargs)
if response:
timestamps = dict(
[(int(post['from']['id']), datetime_parse(post['created_time'])) for post in response['data']])
ids_new = timestamps.keys()
# becouse we should use local pk, instead of remote, remove it after pk -> graph_id
ids_current = map(int, User.objects.filter(pk__in=self.shares_users.get_query_set(
only_pk=True).using(MASTER_DATABASE).exclude(time_from=None)).values_list('graph_id', flat=True))
ids_add = set(ids_new).difference(set(ids_current))
ids_add_pairs = []
ids_remove = set(ids_current).difference(set(ids_new))
log.debug('response objects count=%s, limit=%s, after=%s' %
(len(response['data']), limit, kwargs.get('after')))
for post in response['data']:
graph_id = int(post['from']['id'])
if sorted(post['from'].keys()) == ['id', 'name']:
try:
user = get_or_create_from_small_resource(post['from'])
ids += [user.pk]
# this id in add list and still not in add_pairs (sometimes in response are duplicates)
if graph_id in ids_add and graph_id not in map(lambda i: i[0], ids_add_pairs):
# becouse we should use local pk, instead of remote
ids_add_pairs += [(graph_id, user.pk)]
except UnknownResourceType:
continue
m2m_model = self.shares_users.through
# '(album|post)_id'
field_name = [f.attname for f in m2m_model._meta.local_fields
if isinstance(f, models.ForeignKey) and f.name != 'user'][0]
# remove old shares without time_from
self.shares_users.get_query_set_through().filter(time_from=None).delete()
# in case some ids_add already left
self.shares_users.get_query_set_through().filter(
**{field_name: self.pk, 'user_id__in': map(lambda i: i[1], ids_add_pairs)}).delete()
# add new shares with specified `time_from` value
get_share_date = lambda id: timestamps[id] if id in timestamps else self.created_time
m2m_model.objects.bulk_create([m2m_model(
**{field_name: self.pk, 'user_id': pk, 'time_from': get_share_date(graph_id)}) for graph_id, pk in ids_add_pairs])
return User.objects.filter(pk__in=ids), response
def test_dump(self):
aero_file = AeroFile(test_path / "data_files" / "aero_1.yaml")
aero_file.dump(test_path / "data_files" / "aero_2.i.yaml")
aero_1_data = flex_load(test_path / "data_files" / "aero_1.yaml")
aero_2_data = flex_load(test_path / "data_files" / "aero_2.i.yaml")
# Check whether the data in the aero_2_data dictionary is the same as in aero_1_data.
# Since aero_1_data was written by hand, it doesn't contain all the optional parameters
# for the coefficient models, while aero_2_data does. So we are only checking the parameters
# in aero_1_data and ignore any parameter unique to aero_2_data.
def check_dictionaries(d1, d2):
for key, value in d1.items():
# print(f"dict key='{key}, v1='{value}, v2={d2[key]}")
if isinstance(value, dict):
self.assertIsInstance(d2[key], dict)
check_dictionaries(value, d2[key])
elif isinstance(value, float):
self.assertAlmostEqual(value, d2[key])
elif isinstance(value, (list, tuple)):
self.assertIsInstance(d2[key], (list, tuple))
check_list(value, d2[key])
elif isinstance(value, set):
# It's assumed that sets do not contain lists, dictionaries or
# other sets and that both sets should contain the same elements.
self.assertIsInstance(d2[key], set)
self.assertEqual(len(d2[key] - value), 0)
else:
self.assertEqual(value, d2[key])
def check_list(l1, l2):
for i, value in enumerate(l1):
# print(f"list i='{i}, v1='{value}, v2={l2[i]}")
if isinstance(value, dict):
self.assertIsInstance(l2[i], dict)
check_dictionaries(value, l2[i])
elif isinstance(value, float):
self.assertAlmostEqual(value, l2[i])
elif isinstance(value, (list, tuple)):
self.assertIsInstance(l2[i], (list, tuple))
check_list(value, l2[i])
elif isinstance(value, set):
# It's assumed that sets do not contain lists, dictionaries or
# other sets and that both sets should contain the same elements.
self.assertIsInstance(l2[i], set)
self.assertEqual(len(l2[i] - value), 0)
else:
self.assertEqual(value, l2[i])
aero_2_data['created'] = datetime_parse(aero_2_data['created'])
check_dictionaries(aero_1_data, aero_2_data)
def get(self, request):
url_logs = URLStatusLog.objects.all().values('publication').order_by('publication', '-last_modified'). \
annotate(last_modified=Max('last_modified')). \
values_list('publication', 'type', 'publication__date_published_text').order_by('publication')
all_records = Counter()
years = []
if url_logs:
start_year = 1900
end_year = 2100
if request.query_params.get('start_date'):
start_year = int(request.query_params.get('start_date'))
if request.query_params.get('end_date'):
end_year = int(request.query_params.get('end_date'))
for pub, category, date in url_logs:
try:
date_published = int(datetime_parse(str(date)).year)
except:
date_published = None
if date_published is not None and start_year <= date_published <= end_year:
years.append(date_published)
all_records[(date_published, category)] += 1
else:
sqs = SearchQuerySet()
sqs = sqs.filter(**visualization_query_filter(request))
filtered_pubs = queryset_gen(sqs)
pubs = Publication.api.primary(pk__in=filtered_pubs)
for pub in pubs:
if pub.code_archive_url is not '' and pub.year_published is not None:
years.append(pub.year_published)
all_records[(pub.year_published,
categorize_url(pub.code_archive_url))] += 1
group = []
data = [['x']]
for name in URLStatusLog.PLATFORM_TYPES:
group.append(name[0])
data.append([name[0]])
for year in sorted(set(years)):
data[0].append(year)
index = 1
for name in URLStatusLog.PLATFORM_TYPES:
data[index].append(all_records[(year, name[0])])
index += 1
return Response(
{
"aggregated_data": json.dumps(data),
"group": json.dumps(group)
},
template_name="visualization/code_archived_url_staged_bar.html")
def _detect_autoscaler(self, minutes=10):
"""Grep syslog for autoscaler.
Also get remote's date (time), to avoid another (slow) ssh."""
command = """grep CRON /var/log/syslog | grep autoscaler | tail; date"""
output = self.remote_cmd(command).splitlines()
recent_crons, now = output[:-1], output[-1]
if not recent_crons:
return False
# Get timestamp of last cron job
last_cron = recent_crons[-1].split(self.name)[0]
log_time = datetime_parse(last_cron).replace(tzinfo=timezone.utc)
now = datetime_parse(now)
pause = timedelta(minutes=minutes)
if log_time + pause < now:
return False
return True
def main():
"""Shows basic usage of the Sheets API.
Creates a Sheets API service object and prints the names and majors of
students in a sample spreadsheet:
https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/edit
"""
discoveryUrl = ('https://sheets.googleapis.com/$discovery/rest?'
'version=v4')
service = discovery.build(
'sheets',
'v4',
developerKey="AIzaSyCZ41Zj4_eChwYANXzbDBajEWX8dt3a2Ao",
discoveryServiceUrl=discoveryUrl)
spreadsheetId = '1zcCmtkVP4Hjy5uDuN4veMTqYquMRcF-82mMt2kRhnmU'
rangeName = 'Sheet1!A2:E'
result = service.spreadsheets().values().get(
spreadsheetId=spreadsheetId, range=rangeName).execute()
values = result.get('values', [])
if not values:
print('No data found.')
else:
for row in fix_rows(values):
row[0] = datetime_parse(row[0])
if row[2]:
row[2] = datetime_parse(row[2])
print(
'new Milestone("{name}", "{deadline}", {remove}, {marker}, {shift}),'
.format(
name=row[1],
deadline=row[0].strftime("%Y-%m-%d %H:%M"),
remove=row[2].strftime('"%Y-%m-%d %H:%M"')
if row[2] else "undefined",
marker=f'{row[3]}' if row[3] else "undefined",
shift=f'{row[4]}' if row[4] else "undefined",
))
def attribute_list(request):
organism_id = request.REQUEST.get('organism_id', '')
if not organism_id:
organisms = cache.get('organisms', [])
if organisms:
organism_id = organisms[0][0]
else:
query_organism = {
"method" : "get_organisms",
"key": request.user.sessionkey,
}
organisms_dict = api_request(query_organism)
try:
organism_id = organisms_dict['result']['organisms'][0]['id']
except KeyError:
if organisms_dict.has_key('error'):
msg = organisms_dict['error']['message']
messages.error(request, 'API ERROR: {}. {}'.format(msg, organisms_dict['error']['data']))
else:
raise Http404
query_dict = {
"method" : "get_attributes",
"key": request.user.sessionkey,
"params" : {
"query" : "organism = {}".format(organism_id),
# "limit" : int,
# "skip" : int,
"orderby" : [["name", "asc"]]
}
}
content_dict = api_request(query_dict)
if content_dict.has_key('result'):
attr_list = content_dict['result']['attributes']
for attr in attr_list:
time_value = datetime_parse(attr['created'])
attr['created'] = time_value.strftime("%Y-%m-%d %H:%M:%S")
template_context = {'item_list': attr_list}
else:
template_context ={}
msg = content_dict['error']['message']
messages.error(request, 'API ERROR: {}. {}'.format(msg, content_dict['error']['data']))
return render_to_response("attribute_list.html", template_context, context_instance=RequestContext(request))
def get_local_metadata(self):
"""
Retrieves a local file that records past downloads and returns
a dictionary that includes a timestamp with a timestamp marking
the last update.
If no file exists it returns the dictionary with null values.
"""
metadata = {
'last-download': None
}
if os.path.isfile(self.zip_metadata_path):
with open(self.zip_metadata_path) as f:
metadata['last-download'] = datetime_parse(f.readline())
return metadata
def get(self, request):
url_logs = URLStatusLog.objects.all().values('publication').order_by('publication', '-last_modified'). \
annotate(last_modified=Max('last_modified')). \
values_list('publication', 'type', 'publication__date_published_text').order_by('publication')
all_records = Counter()
years = []
if url_logs:
start_year = 1900
end_year = 2100
if request.query_params.get('start_date'):
start_year = int(request.query_params.get('start_date'))
if request.query_params.get('end_date'):
end_year = int(request.query_params.get('end_date'))
for pub, category, date in url_logs:
try:
date_published = int(datetime_parse(str(date)).year)
except:
date_published = None
if date_published is not None and start_year <= date_published <= end_year:
years.append(date_published)
all_records[(date_published, category)] += 1
else:
sqs = SearchQuerySet()
sqs = sqs.filter(**visualization_query_filter(request))
filtered_pubs = queryset_gen(sqs)
pubs = Publication.api.primary(pk__in=filtered_pubs)
for pub in pubs:
if pub.code_archive_url is not '' and pub.year_published is not None:
years.append(pub.year_published)
all_records[(pub.year_published, CodeArchiveUrl.categorize_url(pub.code_archive_url))] += 1
group = []
data = [['x']]
for name in URLStatusLog.PLATFORM_TYPES:
group.append(name[0])
data.append([name[0]])
for year in sorted(set(years)):
data[0].append(year)
index = 1
for name in URLStatusLog.PLATFORM_TYPES:
data[index].append(all_records[(year, name[0])])
index += 1
return Response({"aggregated_data": json.dumps(data), "group": json.dumps(group)},
template_name="visualization/code_archived_url_staged_bar.html")
def parse_sunset(sunset, date=None, **kwargs):
if isinstance(sunset, datetime):
return sunset
def f(v, date=date):
if not date:
date = datetime.utcnow()
return date + timedelta(seconds=int(sunset))
if isinstance(sunset, str) and sunset.strip().isdigit():
return f(sunset.strip())
if isinstance(sunset, int):
return f(sunset)
return datetime_parse(sunset)
def process_value(cls, value, python_type):
"""
process_value returns a datetime, date
or time object according to a given string
value and a python type.
"""
if not dateutil:
raise ImproperlyConfigured(
"'python-dateutil' is required to process datetimes"
)
return_value = datetime_parse(value)
if issubclass(python_type, datetime.datetime):
return return_value
elif issubclass(python_type, datetime.time):
return return_value.time()
elif issubclass(python_type, datetime.date):
return return_value.date()
def upload(self, blob):
'''
Upload file to the system
@param blob: a file content to be store
@return: file id
'''
res = ServiceResult()
log.info("uploading")
h = blob_hash(blob)
dst = os.path.join(self.upload_dir, h)
with self.get_lock():
with open(dst, 'wb') as f:
f.write(blob)
creation_time = datetime_parse(time.ctime(os.path.getctime(dst)))
# TODO: remove, use dircache instead
valid_until = creation_time + self.file_lifetime
res.result = FileId.construct(self, dst, h)
return res
def get_valid_date(self):
if self._file_id['valid_until'] == -1:
return float("infinity")
else:
return datetime_parse(self._file_id['valid_until'])
def get_events(url, api_key, limit, since=None, to=None):
"""
Get issue's events by URL.
Handle pagination automatically -
https://docs.getsentry.com/on-premise/api/pagination/.
:param url: URL to get events
:type: str
:param key: API key
:type: str
:param limit: Maximum number of events to return
:type: int
:param since: event's min creation datetime
:type: datetime
:param to: event's max creation datetime
:type: datetime
:rtype: list
"""
response = requests.get(url, auth=(api_key, ''))
if not response.ok:
code = response.status_code
if code == 404:
logger.error('Issue not found')
return []
detail = response.json()['detail']
logger.error('Server returned %d: %s', code, detail)
return []
if (since or to) is not None:
events = []
for event in response.json():
created = datetime_parse(event['dateCreated'])
if since is not None and created < since:
# Set limit to avoid asking for more pages.
limit = len(events)
break
if to is not None and created > to:
continue
events.append(event)
else:
events = response.json()
limit -= len(events)
if limit < 0:
events = events[:limit]
limit = 0
if limit > 0 and response.links['next']['results'] == 'true':
events.extend(get_events(url=response.links['next']['url'],
api_key=api_key,
limit=limit,
since=since,
to=to))
return events
def __prepare_date(self, date):
if not date:
return ""
if type(date) is str:
date = datetime_parse(date)
return date.strftime("%Y-%m-%d")
def get_objects(request):
paginate_by = 10
template_context = {}
field_filters_dict_sort = {'': []}
raw_query_str=''
fields = OBJECT_FIELDS
saved_query_list = SavedQuery.objects.filter(user=request.user)
query_history = []
if request.method == 'POST':
form = SelectObjects(request=request, data = request.POST)
query_history = ast.literal_eval(request.POST['query_history'])
# Load from history
query_history_step = request.POST['query_history_step']
if query_history_step:
where_search = ''
field_filters_dict_sort={}
all_attr_type_dict = { key: atype for (key, item, atype) in form.fields['attributes_list'].choices }
all_attr_type_dict.update(OBJECT_FIELDS_CHOICES_WITH_TYPE)
if re.findall('\(.+\)', query_history_step):
old_raw_query_str, query_step_st = re.findall('(\(.+\)) AND (.+)', query_history_step)[0]
else:
old_raw_query_str = ''
query_step_st = query_history_step
raw_query_str = query_history_step
if query_step_st:
if ' AND ' in query_step_st:
logic_rel = ' AND '
logic_operation = 'ALL'
else:
logic_rel = ' OR '
logic_operation = 'ANY'
for i, attr in enumerate(query_step_st.split(logic_rel)):
if attr.find('"') != -1:
attr_name, operation, attr_value = re.findall('(.+) (.+) (".+")', attr)[0]
else:
attr_name, operation, attr_value = attr.split()
_attr_name = attr_name.replace('attr.', '') if attr_name.startswith('attr.') else attr_name
field_filters_dict_sort[i] = (attr_name, operation, attr_value.replace('"', ''), all_attr_type_dict[_attr_name])
else:
where_search = request.POST.get('where_search')
logic_operation = request.POST.get('select_operand_in') if where_search == 'search_in_results' else request.POST.get('select_operand')
raw_query_str = request.POST['raw_query_str']
old_raw_query_re = re.findall('\(.+\)', raw_query_str)
old_raw_query_str = old_raw_query_re[0] if old_raw_query_re else ''
field_filters_dict = ast.literal_eval(request.POST['field_filters_dict'])
if field_filters_dict:
field_filters_dict_sort={}
for key, q in field_filters_dict.items():
if key:
q[1] = mark_safe(q[1])
field_filters_dict_sort[int(key)] = [ s.decode('utf8') for s in q ]
template_context.update({
'logic_operation': logic_operation,
'raw_query_str': raw_query_str,
'old_raw_query_str': old_raw_query_str,
'query_history_step': query_history_step
})
if form.is_valid():
cd = form.cleaned_data
paginate_by = int(cd['paginate_by'])
raw_query = 'organism = {}'.format(cd['organism'])
if cd['tags']:
raw_query = raw_query + ' & tags.contains("{}")'.format('","'.join(cd['tags']))
if raw_query_str:
prep_raw_query_str = raw_query_str.replace(' AND ', ' & ').replace(' OR ', ' | ')
raw_query = raw_query + ' & (' + prep_raw_query_str + ')'
attr_list=[]
attr_list = cd['attributes_list']
order_field = request.GET.get('order_by', 'name')
order_field = cd['sort_by'] if cd['sort_by'] in OBJECT_FIELDS else 'attr.' + cd['sort_by']
query_dict = {
"method" : 'get_objects',
"key": request.user.sessionkey,
"params" : {
"count": 'true',
"query" : raw_query,
"nulls_filler": "n/a",
# "limit" : int,
# "skip": int,
"orderby" : [(order_field, "asc"),],
"attributes_list": cd['attributes_list']
}
}
try:
content_dict = get_pagination_page(page=1, paginate_by=paginate_by, query_dict=query_dict)
except socket.error:
# TODO
messages.error(request, 'Oops! Not connected to server.')
return render_to_response("select_objects.html", template_context, context_instance=RequestContext(request))
if content_dict.has_key('result'):
if len(cd['display_fields']):
display_fields = cd['display_fields']
else:
display_fields = fields
objects_count = content_dict['result']['total_count']
object_list = []
for obj in content_dict['result']['objects']:
object_fields=[]
for field in display_fields:
if field in ('created', 'modified'):
time_value = datetime_parse(obj[field])
field_value = time_value.strftime("%Y-%m-%d %H:%M:%S")
object_fields.append( (field, field_value) )
else:
object_fields.append( (field, obj[field]) )
if obj.has_key('attributes'):
object_attrs = [ None for i in attr_list ]
for obj_attr in obj['attributes']:
attr_index = attr_list.index(obj_attr['name'])
object_attrs[attr_index] = obj_attr
else:
object_attrs = []
object_list.append(
{'object_name': obj['name'],
'url': reverse('update_object', kwargs={'object_id': obj['id']}),
'fields': object_fields,
# 'attrs': [ d for attr in attr_list for d in obj['attributes'] if d['name'] == attr ]
'attrs': object_attrs,
})
previous_page = False
next_page = True if int(objects_count) > paginate_by else False
pages_count = int(math.ceil(objects_count/paginate_by))
if pages_count < 12:
pages = range(1, pages_count+1)
else:
pages = range(1, 4)
pages.append('...')
pages.extend(range(pages_count-2, pages_count+1))
if query_history:
if query_history_step:
# Loading from history
for step in query_history:
if step['query'] == query_history_step.encode('utf8'):
step['count'] = objects_count
idx = query_history.index(step)
query_history = query_history[:idx+1]
break
elif where_search == 'search_in_results' and query_history[-1]['query'] != raw_query_str.encode('utf8'):
# Search in result
query_history.append({'query': raw_query_str, 'count': objects_count})
else:
# Change current query or new query
query_history[-1] = {'query': raw_query_str, 'count': objects_count}
else:
query_history.append({'query': raw_query_str, 'count': objects_count})
display_fields_str = mark_safe(json.dumps(display_fields))
template_context.update({
'display_fields': display_fields,
'display_fields_str': display_fields_str,
'attributes': attr_list,
'object_list': object_list,
'object_download_form': DownloadForm(),
'has_next': next_page,
'has_previous': previous_page,
'pages': pages,
'this_page': 1,
'next_page_number': 2,
'paginate_by': paginate_by,
'items_count': objects_count,
})
else:
msg = content_dict['error']['message']
messages.error(request,
'API ERROR: {}. {}'.format(msg, content_dict['error']['data']))
query_dict_str = mark_safe(json.dumps(query_dict))
template_context.update({
'query_dict_str': query_dict_str
})
else:
# TODO print validation error
# For saved query
elif request.method == 'GET' and request.GET.get('saved_query', None):
query_name = request.GET['saved_query']
saved_query = SavedQuery.objects.get(name=query_name)
form_data = {
'organism': saved_query.organism_id,
'display_fields': saved_query.display_fields,
'attributes_list': saved_query.attributes_list,
'paginate_by': saved_query.paginate_by,
'sort_by': saved_query.sort_by
}
form = SelectObjects(request=request, data=form_data)
if saved_query.filter_fields.items():
field_filters_dict_sort = {}
for key, q in saved_query.filter_fields.items():
if key:
q[1] = mark_safe(q[1])
field_filters_dict_sort[int(key)] = q
raw_query_str = saved_query.query_str
old_raw_query_re = re.findall('\(.+\)', raw_query_str)
old_raw_query_str = old_raw_query_re[0] if old_raw_query_re else ''
if raw_query_str:
step = True
raw_query_str_iter = raw_query_str
query_history.append({'query': raw_query_str, 'count': ''})
while step:
_query_re = re.findall('\((.+)\)', raw_query_str_iter)
step = _query_re[0] if _query_re else None
if step:
query_history.append({'query': step, 'count': ''})
raw_query_str_iter = step
query_history.reverse()
template_context.update({
'logic_operation': "ALL",
'raw_query_str': raw_query_str,
'old_raw_query_str': old_raw_query_str,
})
else:
form = SelectObjects(request=request)
attributes_from_organism = form.fields['attributes_list'].choices
template_context.update({
'form': form,
'method': 'get_objects',
'saved_query_list': saved_query_list,
'fields': fields,
'fields_with_type': OBJECT_FIELDS_CHOICES_WITH_TYPE,
'field_filters_dict': field_filters_dict_sort,
'attributes_from_organism': attributes_from_organism,
'query_history': query_history
})
return render_to_response("select_objects.html", template_context, context_instance=RequestContext(request))
def pagination(request, page, paginate_by, items_count, data):
data = json.loads(data)
display_fields = data['display_fields']
query_dict = data['query_dict']
query_dict['params']['count'] = False
if query_dict['params'].has_key('attributes_list'):
attributes = [ attr for attr in query_dict['params']['attributes_list'] ]
else:
attributes = []
content_dict = get_pagination_page(page=page, paginate_by=paginate_by, query_dict=query_dict)
if content_dict.has_key('result'):
object_list = []
for obj in content_dict['result']['objects']:
object_fields=[]
for field in display_fields:
if field in ('created', 'modified'):
time_value = datetime_parse(obj[field])
field_value = time_value.strftime("%Y-%m-%d %H:%M:%S")
object_fields.append( (field, field_value) )
else:
object_fields.append( (field, obj[field]) )
if obj.has_key('attributes'):
object_attrs = [ None for i in attributes ]
for obj_attr in obj['attributes']:
attr_index = attributes.index(obj_attr['name'])
object_attrs[attr_index] = obj_attr
else:
object_attrs = []
object_list.append(
{'object_name': obj['name'],
'url': reverse('update_object', kwargs={'object_id': obj['id']}),
'fields': object_fields,
'attrs': object_attrs
}
)
next_page = True if paginate_by * page < items_count else False
previous_page = True if page > 1 else False
pages_count = int(math.ceil(items_count/paginate_by))
if pages_count <= 11:
pages = range(1, pages_count+1)
elif page == 1 or page == pages_count:
pages = range(1, 4)
pages.append('...')
pages.extend(range(pages_count-2, pages_count+1))
elif 1 < page <= 6:
# page in begin
pages = range(1, page+2)
pages.append('...')
pages.extend(range(pages_count-2, pages_count+1))
elif pages_count-5 <= page < pages_count:
# page in end
pages = range(1, 4)
pages.append('...')
pages.extend(range(page-1, pages_count+1))
else:
# page in middle
pages = range(1, 4)
pages.append('...')
pages.extend(range(page-1, page+2))
pages.append('...')
pages.extend(range(pages_count-2, pages_count+1))
query_dict_str = mark_safe(simplejson.dumps(query_dict))
display_fields_str = mark_safe(simplejson.dumps(display_fields))
template_context = {
'has_next': next_page,
'has_previous': previous_page,
'next_page_number': page+1,
'previous_page_number': page-1,
'paginate_by': paginate_by,
'pages': pages,
'this_page': page,
'items_count': items_count,
'display_fields': display_fields,
'display_fields_str': display_fields_str,
'attributes': attributes,
'object_list': object_list,
'query_dict_str': query_dict_str
}
render = render_to_string('object_list.html', template_context)
dajax = Dajax()
dajax.assign('#js_object_result_table', 'innerHTML', render)
dajax.script('stop_show_loading();')
return dajax.json()
def get(self, instance, owner):
val_str = instance.element.get(self.attrib_name)
if val_str is None:
return self.get_optional(instance)
else:
return datetime_parse(val_str)