def get_allowed_time_periods(self): time_periods = [] if self.allowed_times: for p in self.allowed_times.split('\n'): if p: start, end = p.split(' > ') try: time_periods.append( period( parse_date(start.strip()), parse_date(end.strip()), ) ) # If someone has entered garbage, dump the lot except ValueError as e: time_periods = [] break # If we've not overridden it, use the user-specified periods if not time_periods and self.available_times: for p in self.available_times.split(','): if p: time_periods.append(timeslot_to_period(p.strip(), type=self.type)) time_periods = self.fix_hard_time_limits(time_periods) return make_periods_contiguous(time_periods)
def valid_date(key, config_date): """ traildb bucket folders are not zero-padded so this validation checks that the keys returned by the paginator are *really after* the config date """ key_date = "/".join(key.split("/")[4:7]) return parse_date(key_date) > parse_date(config_date)
def get_json_events(url): f = urllib.urlopen(url) raw_json = f.read() f.close() events_json = json.loads(raw_json) events = events_json.get('events', []) events = [event for event in events if upcoming(event)] items = [] if len(events) > 0: events.sort(key = lambda e: parse_date(e.get('date'))) for event in events: title = event.get('title', 'Untitled event') start_date = parse_date(event.get('date')) event_date_str = make_date_range(start_date, start_date) # only handle single day events for now event_tags = event.get('tags', []) event_url = event.get('url', None) entry = make_link(title, event_url) + "<br>" entry += event_date_str if len(event_tags) > 0: entry += "<br>" entry += " ".join([make_tag(tag) for tag in event_tags]) items.append(entry) return make_ul(items) else: return "<p>No upcoming events</p>"
def get_category(): category = request.args.get('category') date_from = request.args.get('date_from') date_to = request.args.get('date_to') currency = request.args.get('currency') or app.config['DEFAULT_CURRENCY'] if not (category and date_from and date_to): return abort(400) date_from = parse_date(date_from) date_to = parse_date(date_to) payments = Payment.query.filter_by(category=category).\ filter(Payment.date.between(date_from, date_to)).\ filter(Payment.amount > 0).\ order_by(Payment.date.desc()).all() response = [] for payment in payments: item = { 'date': payment.date.strftime('%Y-%m-%d'), 'amount': '%.2f' % payment.convert_to(currency), 'description': payment.description } response.append(item) return json.dumps(response)
def add_arguments(self, parser): parser.add_argument('dataset', help='The id of the dataset') parser.add_argument('start_date', type = lambda s: parse_date(s), help='The start date') parser.add_argument('end_date', nargs = '?', type = lambda s: parse_date(s), default = datetime.utcnow(), help='The end date') parser.add_argument('--update', default = False, action='store_true', help='Update metadata even if already present in DB') parser.add_argument('--tags', default = [], nargs='*', help='A list of tag names to set to the metadata') parser.add_argument('--debug', default = False, action='store_true', help='Show debugging info')
def process_commit(c, r, metrics, stats, since, now): # Find the oldest of the pr commit/dates # TODO find commit dates commit_date = parse_date(max(filter( None, ( c['authoredDate'], c['committedDate'], c['pushedDate'])))) if commit_date < since: return found = False for status_ctx in c['status']['contexts']: if status_ctx['context'] == metrics.dims['Hook']: status_time = parse_date(status_ctx['createdAt']) found = True if found: tdelta = (status_time - commit_date).total_seconds() metrics.put_metric('RepoHookLatency', tdelta, 'Seconds') stats['runtime'] += tdelta stats['count'] += 1 else: stats['missing'] += 1 stats['missing_time'] += (now - commit_date).total_seconds()
def _make_issue(self, project_id, issue_number, json): d = {} d['number'] = issue_number d['title'] = json['title'] d['body'] = json['body'].strip() d['creation'] = parse_date(json['created_at']) d['updated'] = parse_date(json['updated_at']) d['author'] = json['user']['login'] d['status'] = json['state'] d['url'] = 'https://github.com/%s/issues/%s' % (project_id, issue_number) if json['assignee']: d['assignee'] = json['assignee']['login'] else: d['assignee'] = None if json['milestone']: d['version'] = json['milestone'] else: d['version'] = None d['has_comments'] = (json['comments'] > 0) d['attachments'] = list(self._extract_attachments(d['body'])) d['labels'] = json['labels'] # TODO fetch other updates? return d
def create_calendar(self, data, **kwargs): for row in data: try: id = int(row['service_id']) except ValueError: print 'WARNING: Could not process service id of %s' % row continue begin, end = None, None if row['start_date']: begin = parse_date(row['start_date']) if row['end_date']: end = parse_date(row['end_date']) defaults = { 'begin': begin, 'end': end } for dow in ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']: defaults[dow] = str(row[dow]) == '1' try: Schedule(id=id, **defaults).save() except: print 'Schedule exists' continue
def setUp(self): super(VolumeTests, self).setUp() self.test_vol_1_box = BoxVolume(self.test_project_id, self.test_user_id, { 'title': 'Test volume 1', 'type': 'box', 'comment': 'Comment on test volume 1', 'min_x': -1, 'min_y': -1, 'min_z': -1, 'max_x': 1, 'max_y': 1, 'max_z': 1 }) self.test_vol_1_id = self.test_vol_1_box.save() cursor = connection.cursor() cursor.execute(""" SELECT row_to_json(v) FROM ( SELECT id, project_id, name, comment, user_id, editor_id, creation_time, edition_time, Box3D(geometry) as bbox, ST_Asx3D(geometry) as geometry FROM catmaid_volume v ) v """) self.test_vol_1_data = cursor.fetchall()[0][0] self.test_vol_1_data['creation_time'] = parse_date(self.test_vol_1_data['creation_time']) self.test_vol_1_data['edition_time'] = parse_date(self.test_vol_1_data['edition_time'])
def _transaction_from_tree(tree, accountdict, commoditydict): trn = '{http://www.gnucash.org/XML/trn}' cmdty = '{http://www.gnucash.org/XML/cmdty}' ts = '{http://www.gnucash.org/XML/ts}' split = '{http://www.gnucash.org/XML/split}' guid = tree.find(trn + "id").text currency_space = tree.find(trn + "currency/" + cmdty + "space").text currency_name = tree.find(trn + "currency/" + cmdty + "id").text currency = commoditydict[(currency_space, currency_name)] date = parse_date(tree.find(trn + "date-posted/" + ts + "date").text) date_entered = parse_date(tree.find(trn + "date-entered/" + ts + "date").text) description = tree.find(trn + "description").text slots = _slots_from_tree(tree.find(trn + "slots")) transaction = Transaction(guid=guid, currency=currency, date=date, date_entered=date_entered, description=description, slots=slots) for subtree in tree.findall(trn + "splits/" + trn + "split"): split = _split_from_tree(subtree, accountdict, transaction) transaction.splits.append(split) return transaction
def _received_in_time(record, extra_data): """Check if publication is not older than 24h """ api_url = current_app.config.get('CROSSREF_API_URL') api_response = requests.get(api_url % get_first_doi(record)) if api_response.status_code != 200: return True, ('Article is not on crossref.', ), 'Api response: %s' % api_response.text details_message = "" api_message = api_response.json()['message'] if 'publication_info' in record and \ record['publication_info'][0]['journal_title'] == 'Progress of Theoretical and Experimental Physics': parts = api_message['published-online']['date-parts'][0] # if we don't have month or day substitute it with 1 if len(parts) < 3: parts.extend([1] * (3 - len(parts))) details_message += 'Month and/or day is missing, substitute it with "1".' # only contains day of publication, check for end of day api_time = datetime(*parts, hour=23, minute=59, second=59) time_source = '"published online" field' else: api_time = parse_date(api_message['created']['date-time'], ignoretz=True) time_source = 'crossref' received_time = parse_date(record['record_creation_date']) delta = received_time - api_time check_accepted = delta <= timedelta(hours=24) details_message += 'Arrived %d hours later then creation date on crossref.org.' % (delta.total_seconds() / 3600) debug = 'Time from %s: %s, Received time: %s' % (time_source, api_time, received_time) return check_accepted, (details_message, ), debug
def parse_date_safely(x): x = str(x) try: return parse_date(x) except ValueError: # http://stackoverflow.com/questions/14595401 return parse_date(x + '-01-01')
def _slots_from_tree(tree): if tree is None: return {} slot = "{http://www.gnucash.org/XML/slot}" ts = "{http://www.gnucash.org/XML/ts}" slots = {} for elt in tree.findall("slot"): key = elt.find(slot + "key").text value = elt.find(slot + "value") type_ = value.get('type', 'string') if type_ in ('integer', 'double'): slots[key] = int(value.text) elif type_ == 'numeric': slots[key] = _parse_number(value.text) elif type_ in ('string', 'guid'): slots[key] = value.text elif type_ == 'gdate': slots[key] = parse_date(value.find("gdate").text) elif type_ == 'timespec': slots[key] = parse_date(value.find(ts + "date").text) elif type_ == 'frame': slots[key] = _slots_from_tree(value) else: raise RuntimeError("Unknown slot type {}".format(type_)) return slots
def ValidTravelPlanned(self, userInfo, request): passFlag = True if len(userInfo['destination']) < 1: messages.warning(request, 'Destination field left blank.') passFlag = False if len(userInfo['description']) < 1: messages.warning(request, 'Description field left blank.') passFlag = False unicode_text_from = userInfo['travel_date_from'] from_date = parse_date(unicode_text_from) if from_date.date() < date.today(): messages.warning(request, 'Travel from date cannot be in the past.') passFlag = False unicode_text_to = userInfo['travel_date_to'] to_date = parse_date(unicode_text_to) if to_date.date() < date.today(): messages.warning(request, 'Travel to date cannot be in the past.') passFlag = False if to_date < from_date: messages.warning(request, 'Travel to date cannot be before travel from date.') passFlag = False if passFlag: logged_in = request.session['logged_in'] travelplanner = User.objects.get(id=logged_in) travelmaker = User.objects.get(id=logged_in) travel = self.create(destination = userInfo['destination'], description = userInfo['description'], travel_date_from = userInfo['travel_date_from'], travel_date_to = userInfo['travel_date_to'], travelplanner_id = travelplanner) return passFlag
def __query_loads__(self, request_query): """ """ # Some fields might express a date # We try to convert those strings to datetime indexes = self.getParentDatabase().getIndex().Indexes request_query = json.loads(request_query) for key, value in request_query.iteritems(): if key in indexes: index = indexes[key] # This is lame: we should check if it quacks, not # if it's a duck! # XXX Use a more robust method to tell apart # date indexes from non-dates if isinstance(index, DateIndex): # convert value(s) to date(s) if isinstance(value, basestring): request_query[key] = parse_date(value) elif 'query' not in value: # it means value is a list of date values to be used # with the implicit default operator query OR request_query[key] = map(parse_date, value) else: # it means value is a dictionary if isinstance(value['query'], basestring): # query got a single comparison value request_query[key]['query'] = parse_date( value['query']) else: # query got a list of comparison values request_query[key]['query'] = map( parse_date, value['query'] ) return request_query
def test_project_creation( self, app, user, public_project, private_project, public_url, private_url): # test_project_created res = app.get(public_url) assert res.status_code == 200 assert len(res.json['data']) == public_project.logs.count() assert public_project.logs.first().action == 'project_created' assert public_project.logs.first( ).action == res.json['data'][API_LATEST]['attributes']['action'] # test_log_create_on_public_project res = app.get(public_url) assert res.status_code == 200 assert len(res.json['data']) == public_project.logs.count() assert_datetime_equal( parse_date( res.json['data'][API_FIRST]['attributes']['date']), public_project.logs.first().date) assert res.json['data'][API_FIRST]['attributes']['action'] == public_project.logs.first( ).action # test_log_create_on_private_project res = app.get(private_url, auth=user.auth) assert res.status_code == 200 assert len(res.json['data']) == public_project.logs.count() assert_datetime_equal( parse_date( res.json['data'][API_FIRST]['attributes']['date']), private_project.logs.first().date) assert res.json['data'][API_FIRST]['attributes']['action'] == private_project.logs.first( ).action
def search(request): if 'q' in request.GET and request.GET['q']: TAG = request.GET['q'] if 'start' in request.GET and request.GET['start']: START = parse_date(request.GET['start']) END = parse_date(request.GET['end']) store_data(TAG, START, END) return render(request, 'webapp/base.html')
def parse(self, filename): tree = etree.parse(filename) root = tree.getroot() for entry in root.findall("default:entry", CveXmlParser.NS): cve = {} cve["cveid"] = entry.find("vuln:cve-id", CveXmlParser.NS).text cve["summary"] = entry.find("vuln:summary", CveXmlParser.NS).text try: cve["cwe"] = entry.find("vuln:cwe", CveXmlParser.NS).get("id") except AttributeError: cve["cwe"] = "" pass cve["published_date"] = parse_date(entry.find("vuln:published-datetime", CveXmlParser.NS).text) cve["modified_date"] = parse_date(entry.find("vuln:last-modified-datetime", CveXmlParser.NS).text) if self.ignore_before_date and cve["modified_date"] < self.ignore_before_date: continue try: cvss_base = entry.find("vuln:cvss", CveXmlParser.NS).find("cvss:base_metrics", CveXmlParser.NS) cve["cvss_score"] = cvss_base.find("cvss:score", CveXmlParser.NS).text cve["cvss"] = "/".join("%s:%s" % (k,cvss_base.find("cvss:%s" % k, CveXmlParser.NS).text) for k in ["access-vector", "access-complexity", "authentication", "confidentiality-impact", "integrity-impact", "availability-impact"]) except AttributeError: cve["cvss_score"] = "-1" cve["cvss"] = "" try: cpes = [] for cpe in entry.find("vuln:vulnerable-software-list", CveXmlParser.NS).findall("vuln:product", CveXmlParser.NS): cpes.append(cpe.text) cve["cpes"] = cpes except AttributeError: cve["cpes"] = "" try: refs = [] for ref in entry.findall("vuln:references", CveXmlParser.NS): ref_obj = {} ref_obj["type"] = ref.get("reference_type") ref_obj["source"] = ref.find("vuln:source", CveXmlParser.NS).text ref_obj["href"] = ref.find("vuln:reference", CveXmlParser.NS).get("href") ref_obj["value"] = ref.find("vuln:reference", CveXmlParser.NS).text refs.append(ref_obj) cve["references"] = refs except AttributeError as e: cve["references"] = [] self.cve_list.append(cve) return self.cve_list
def iter_threads(self): for thread in self.browser.get_threads(): if not "person" in thread: # The account has been removed, probably because it was a # spammer. continue t = Thread(thread["_id"]) t.flags = Thread.IS_DISCUSSION t.title = u"Discussion with %s" % thread["person"]["name"] contact = self.storage.get("contacts", t.id, default={"lastmsg": 0}) birthday = parse_date(thread["person"]["birth_date"]).date() signature = u"Age: %d (%s)" % ((datetime.date.today() - birthday).days / 365.25, birthday) signature += u"\nLast ping: %s" % parse_date(thread["person"]["ping_time"]).strftime("%Y-%m-%d %H:%M:%S") signature += u"\nPhotos:\n\t%s" % "\n\t".join([photo["url"] for photo in thread["person"]["photos"]]) signature += u"\n\n%s" % thread["person"]["bio"] t.root = Message( thread=t, id=1, title=t.title, sender=unicode(thread["person"]["name"]), receivers=[self.browser.my_name], date=parse_date(thread["created_date"]), content=u"Match!", children=[], signature=signature, flags=Message.IS_UNREAD if int(contact["lastmsg"]) < 1 else 0, ) parent = t.root for msg in thread["messages"]: flags = 0 if int(contact["lastmsg"]) < msg["timestamp"]: flags = Message.IS_UNREAD msg = Message( thread=t, id=msg["timestamp"], title=t.title, sender=unicode( self.browser.my_name if msg["from"] == self.browser.my_id else thread["person"]["name"] ), receivers=[ unicode(self.browser.my_name if msg["to"] == self.browser.my_id else thread["person"]["name"]) ], date=parse_date(msg["sent_date"]), content=unicode(msg["message"]), children=[], parent=parent, signature=signature if msg["to"] == self.browser.my_id else u"", flags=flags, ) parent.children.append(msg) parent = msg yield t
def next_build_in(self): # returns the number of seconds until the next build or INFINITY # if the project well never be built (errors, paused, etc) if hasattr(self, 'next_build_at') and self.next_build_at is not None: interval = parse_date(self.next_build_at) - parse_date(self.current_date_time) return interval.total_seconds() else: return INFINITY
def sortbyctime(a, b): da = parse_date(a) db = parse_date(b) if (da > db): return 1 elif da == db: return 0 else: return -1
def process(self, row): _, author_names, editor_names, published, publisher, title, subtitle, url, resource_type, \ keyword_names, abstract, review, fulltext_url, category, discussion, journal, volume, \ number, startpage, endpage, series, edition, sourcetype = row[:23] if Resource.objects.filter(url=url).exists(): logger.info('Skipping existing entry %r', title) return authors = [Person.objects.get_or_create(name=author_name.strip())[0] for author_name in author_names.split(',') if author_name.strip()] editors = [Person.objects.get_or_create(name=editor_name.strip())[0] for editor_name in editor_names.split(',') if editor_name.strip()] keywords = [Keyword.objects.get_or_create(name=keyword_name.strip())[0] for keyword_name in keyword_names.split(',') if keyword_name.strip()] categories = Category.objects.get_or_create(name=category.strip())[:1] if discussion.strip(): review += '\n\nDiscussion: {}'.format(discussion) published = parse_date(published) accessed = parse_date('2015-11-03') resource_type = { 'Academic Paper': models_choices.STUDY, 'Academic Paper (Unpublished)': models_choices.STUDY, 'Blog Post': models_choices.BLOG_ARTICLE, 'Book': models_choices.BOOK, 'Historical Document': models_choices.HISTORICAL_DOCUMENT, 'Industry Publication': models_choices.RESEARCH_SUMMARY, 'Newspaper opinion piece': models_choices.OPINION_PIECE, 'Research Summary': models_choices.RESEARCH_SUMMARY, 'Wikipedia Entry': models_choices.ENCYCLOPEDIA_ARTICLE, '': models_choices.OTHER}[resource_type] resource = Resource( published=published, accessed=accessed, publisher=publisher.strip(), title=title.strip(), subtitle=subtitle.strip(), url=url.strip(), fulltext_url=fulltext_url.strip(), resource_type=resource_type, abstract=abstract.strip(), review=review.strip(), journal=journal.strip(), volume=int(volume.strip()) if volume.strip() else None, number=int(number.strip()) if number.strip() else None, startpage=int(startpage.strip()) if startpage.strip() else None, endpage=int(endpage.strip()) if endpage.strip() else None, series=series.strip(), edition=edition.strip(), sourcetype=sourcetype.strip()) resource.save() resource.authors = authors resource.editors = editors resource.keywords = keywords resource.categories = categories resource.save()
def parse_prometheus(alert, external_url): status = alert.get('status', 'firing') labels = copy(alert['labels']) annotations = copy(alert['annotations']) starts_at = parse_date(alert['startsAt']) if alert['endsAt'] == '0001-01-01T00:00:00Z': ends_at = None else: ends_at = parse_date(alert['endsAt']) if status == 'firing': severity = labels.pop('severity', 'warning') create_time = starts_at elif status == 'resolved': severity = 'normal' create_time = ends_at else: severity = 'unknown' create_time = ends_at or starts_at summary = annotations.pop('summary', None) description = annotations.pop('description', None) text = description or summary or '%s: %s on %s' % (labels['job'], labels['alertname'], labels['instance']) try: timeout = int(labels.pop('timeout', 0)) or None except ValueError: timeout = None if external_url: annotations['externalUrl'] = external_url if 'generatorURL' in alert: annotations['moreInfo'] = '<a href="%s" target="_blank">Prometheus Graph</a>' % alert['generatorURL'] return Alert( resource=labels.pop('exported_instance', None) or labels.pop('instance', 'n/a'), event=labels.pop('alertname'), environment=labels.pop('environment', 'Production'), severity=severity, correlate=labels.pop('correlate').split(',') if 'correlate' in labels else None, service=labels.pop('service', '').split(','), group=labels.pop('job', 'Prometheus'), value=labels.pop('value', None), text=text, attributes=annotations, origin='prometheus/' + labels.pop('monitor', '-'), event_type='prometheusAlert', create_time=create_time.astimezone(tz=pytz.UTC).replace(tzinfo=None), timeout=timeout, raw_data=alert, tags=["%s=%s" % t for t in labels.items()] # any labels left are used for tags )
def fetch_ticket(ticket_id, overwrite=False): ticket = zendesk.ticket_show(id=ticket_id)['ticket'] assignee_id = ticket['assignee_id'] summary = ticket['subject'] description = ticket['description'] url = urllib.parse.urljoin(app.config['ZENDESK_URL'], 'tickets/%d/' % ticket_id) field_ids = app.config['ZENDESK_FIELD_IDS'] custom_fields = fields_to_dict(ticket['custom_fields']) start_date = parse_date(custom_fields[field_ids['start_date']]).date() start_time = parse_date(custom_fields[field_ids['start_time']]).time() end_date = parse_date(custom_fields[field_ids['end_date']]).date() end_time = parse_date(custom_fields[field_ids['end_time']]).time() start = datetime.datetime.combine(start_date, start_time) end = datetime.datetime.combine(end_date, end_time) assignee = zendesk.user_show(id=assignee_id)['user'] timezone = friendly_to_tz(assignee.get('time_zone')) event = { 'summary': summary, 'description': description, 'start': { 'dateTime': start.isoformat(), 'timeZone': timezone }, 'end': { 'dateTime': end.isoformat(), 'timeZone': timezone }, 'source': { 'title': ticket_id, 'url': url } } if not overwrite: event_id = insert_event(assignee_id, event) else: event_id = insert_event(assignee_id, event, ticket_id) redis.hmset('ticket:%s' % ticket_id, { 'event_id': event_id.encode(), 'profile_id': str(assignee_id).encode(), }) redis.set('event:%s' % event_id, str(ticket_id).encode()) return event
def _object_changelog(self, uid, mailbox, msguid, limit=None): """ Query storage for changelog events related to the given UID """ # this requires a user context if not self.env.has_key('REQUEST_USER') or not self.env['REQUEST_USER']: return None # fetch event log from storage eventlog = self.storage.get_events(uid, self._resolve_mailbox_uri(mailbox), msguid, limit) # convert logstash entries into a sane changelog event_op_map = { 'MessageNew': 'APPEND', 'MessageAppend': 'APPEND', 'MessageTrash': 'DELETE', 'MessageMove': 'MOVE', } last_append_uid = 0 result = [] if eventlog is not None: for _log in eventlog: # filter MessageTrash following a MessageAppend event (which is an update operation) if _log['event'] == 'MessageTrash' and last_append_uid > int(_log['uidset']): continue # remember last appended message uid if _log['event'] == 'MessageAppend' and _log.has_key('uidset'): last_append_uid = int(_log['uidset']) # compose log entry to return logentry = { 'rev': _log.get('revision', None), 'op': event_op_map.get(_log['event'], 'UNKNOWN'), 'mailbox': self._convert_mailbox_uri(_log.get('mailbox', None)) } try: timestamp = parse_date(_log['timestamp_utc']) logentry['date'] = timestamp.strftime("%Y-%m-%dT%H:%M:%SZ") except Exception, e: try: timestamp = parse_date(_log['timestamp']) logentry['date'] = timestamp.astimezone(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ") except Exception, e: log.warning("Failed to parse timestamp %r: %r", _log['timestamp'], str(e)) logentry['date'] = _log['timestamp'] logentry['user'] = self._get_user_info(_log) result.append(logentry)
def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION info = self.browser.get_thread(thread.id) for user in info['participants']: if user['user']['fb_id'] is not None: user['user']['fb'] = self.browser.get_facebook(user['user']['fb_id']) if user['user']['id'] == self.browser.my_id: me = HappnContact(user['user']) else: other = HappnContact(user['user']) thread.title = u'Discussion with %s' % other.name contact = self.storage.get('contacts', thread.id, default={'lastmsg_date': '1970-01-01T01:01:01+00:00'}) child = None for msg in info['messages']: flags = 0 if parse_date(contact['lastmsg_date']) < parse_date(msg['creation_date']): flags = Message.IS_UNREAD if msg['sender']['id'] == me.id: sender = me receiver = other else: sender = other receiver = me msg = Message(thread=thread, id=msg['id'], title=thread.title, sender=sender.name, receivers=[receiver.name], date=parse_date(msg['creation_date']), content=msg['message'], children=[], parent=None, signature=sender.get_text(), flags=flags) if child: msg.children.append(child) child.parent = msg child = msg thread.root = child return thread
def _get_dates(self): start_date = request.params.get('start_date') end_date = request.params.get('end_date') if start_date: start_date = parse_date(start_date).date() if end_date: end_date = parse_date(end_date).date() if start_date and end_date: if end_date < start_date: (start_date, end_date) = (end_date, start_date) return (start_date, end_date)
def get_date(txt): # dates in the statement don't have year on them # use st_date to fill in this gap, but be careful # when dates cross new year # ex: st_date = 31 Jan 2011 # dates: 24 Dec & 24 Jan # naive parsing would extract them as Jan & Dec 2011 # instead of Dec 2010 & Jan 2011 date = parse_date(txt, default = st_date) if date > st_date: date = parse_date( txt, default = datetime.date(st_date.year-1, st_date.month, 1)) return date
def parse_item(self, item): guid = self.get_text(item, 'guid') if len(guid) > 25: # date can be extracted from last 25 chars name = guid[:-25] created_at = guid[-25:] else: name = guid created_at = None name = name.replace('_', ' ') description = self.get_text(item, 'title') address = self.get_text(item, 'description') updated_at = self.get_text(item, 'updated') # ensure created_at and updated_at are dates if created_at: try: parse_date(created_at) except ValueError: created_at = None if updated_at: try: parse_date(updated_at) except ValueError: updated_at = None try: lat, lng = self.get_text(item, 'georss:point').split(' ') except IndexError: # detail view lat = self.get_text(item, 'georss:lat') lng = self.get_text(item, 'georss:long') # point object geometry = Point(float(lng), float(lat)) result = { "name": name, "status": None, "address": address, "is_published": True, "user": None, "geometry": geometry, "elev": None, "description": description, "notes": guid, "added": created_at, "updated": updated_at, "data": {} } return result
def is_endTime_after_startTime(startDateTime, endDateTime): if (startDateTime and endDateTime): startTime = startDateTime endTime = endDateTime if type(startDateTime) is types.UnicodeType: startTime = parse_date(startDateTime) if type(endDateTime) is types.UnicodeType: endTime = parse_date(endDateTime) startTime = startTime.replace(tzinfo=None) endTime = endTime.replace(tzinfo=None) return endTime >= startTime return True
def workload_status_since(self): """Get the time when the `workload_status` was last updated. """ return parse_date(self.safe_data['workload-status']['since'])
def agent_status_since(self): """Get the time when the `agent_status` was last updated. """ return parse_date(self.safe_data['agent-status']['since'])
def change_date_format(t): return parse_date(t).strftime('%Y-%m-%d %H:%M:%S')
def test_bid_has_custom_values(self, time, date): bid_id = client.create_bid(76137, time=time, date=date) created_bid = client.get_bid(bid_id) expected = parse_date(time + " " + date, dayfirst=True).astimezone(datetime.timezone.utc) self.assertEqual(expected, created_bid.start_time)
def datapusher_hook(context, data_dict): ''' Update datapusher task. This action is typically called by the datapusher whenever the status of a job changes. :param metadata: metadata produced by datapuser service must have resource_id property. :type metadata: dict :param status: status of the job from the datapusher service :type status: string ''' metadata, status = _get_or_bust(data_dict, ['metadata', 'status']) res_id = _get_or_bust(metadata, 'resource_id') # Pass metadata, not data_dict, as it contains the resource id needed # on the auth checks p.toolkit.check_access('datapusher_submit', context, metadata) task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': res_id, 'task_type': 'datapusher', 'key': 'datapusher' }) task['state'] = status task['last_updated'] = str(datetime.datetime.utcnow()) resubmit = False if status == 'complete': # Create default views for resource if necessary (only the ones that # require data to be in the DataStore) resource_dict = p.toolkit.get_action('resource_show')( context, {'id': res_id}) dataset_dict = p.toolkit.get_action('package_show')( context, {'id': resource_dict['package_id']}) for plugin in p.PluginImplementations(interfaces.IDataPusher): plugin.after_upload(context, resource_dict, dataset_dict) logic.get_action('resource_create_default_resource_views')( context, { 'resource': resource_dict, 'package': dataset_dict, 'create_datastore_views': True, }) # Check if the uploaded file has been modified in the meantime if (resource_dict.get('last_modified') and metadata.get('task_created')): try: last_modified_datetime = parse_date( resource_dict['last_modified']) task_created_datetime = parse_date(metadata['task_created']) if last_modified_datetime > task_created_datetime: log.debug('Uploaded file more recent: {0} > {1}'.format( last_modified_datetime, task_created_datetime)) resubmit = True except ValueError: pass # Check if the URL of the file has been modified in the meantime elif (resource_dict.get('url') and metadata.get('original_url') and resource_dict['url'] != metadata['original_url']): log.debug('URLs are different: {0} != {1}'.format( resource_dict['url'], metadata['original_url'])) resubmit = True context['ignore_auth'] = True p.toolkit.get_action('task_status_update')(context, task) if resubmit: log.debug('Resource {0} has been modified, ' 'resubmitting to DataPusher'.format(res_id)) p.toolkit.get_action('datapusher_submit')( context, {'resource_id': res_id})
def create_registration_report(self, before=None, since=None, report_type=None): """ Create a new registration report. `Args:` before: str Limit to registrations that were started before this date, in ISO format (e.g. 2020-01-01) since: str Limit to registrations that were started since this date, in ISO format (e.g. 2020-01-01) report_type: str The type of report to create. If left as None, it creates the default report. The ``extended`` report includes additional fields. Currently only accepts ``extended``. `Returns:` int The ID of the created report. """ report_url = f'registrant_reports.json' # Create the report for the new data report_parameters = { 'partner_id': self.partner_id, 'partner_API_key': self.partner_api_key, } # Declare these here so the logging doesn't error out since_date = before_date = None if report_type: if report_type not in VALID_REPORT_TYPES: raise RTVFailure( f"Invalid report type. Must be one of {VALID_REPORT_TYPES}" ) report_parameters["report_type"] = report_type if since: since_date = parse_date(since).strftime(DATETIME_FORMAT) report_parameters['since'] = since_date if before: before_date = parse_date(before).strftime(DATETIME_FORMAT) report_parameters['before'] = before_date # The report parameters get passed into the request as JSON in the body # of the request. report_str = f"{report_type} report" if report_type else "report" logger.info(f"Creating {report_str} for {self.partner_id} " f"for dates: {since_date} to {before_date}...") response = self.client.request(report_url, 'post', json=report_parameters) if response.status_code != requests.codes.ok: raise RTVFailure("Couldn't create RTV registrations report") response_json = response.json() # The RTV API says the response should include the report_id, but I have not found # that to be the case report_id = response_json.get('report_id') if report_id: logger.info(f"Created report with id {report_id}.") return report_id # If the response didn't include the report_id, then we will parse it out of the URL. status_url = response_json.get('status_url') url_match = STATUS_URL_PARSE_REGEX.search(status_url) if url_match: report_id = url_match.group(1) logger.info(f"Created report with id {report_id}.") return report_id
def xloader_hook(context, data_dict): ''' Update xloader task. This action is typically called by ckanext-xloader whenever the status of a job changes. :param metadata: metadata provided when submitting job. key-value pairs. Must have resource_id property. :type metadata: dict :param status: status of the job from the xloader service. Allowed values: pending, running, running_but_viewable, complete, error (which must all be valid values for task_status too) :type status: string :param error: Error raised during job execution :type error: string NB here are other params which are in the equivalent object in ckan-service-provider (from job_status): :param sent_data: Input data for job :type sent_data: json encodable data :param job_id: An identifier for the job :type job_id: string :param result_url: Callback url :type result_url: url string :param data: Results from job. :type data: json encodable data :param requested_timestamp: Time the job started :type requested_timestamp: timestamp :param finished_timestamp: Time the job finished :type finished_timestamp: timestamp ''' metadata, status = _get_or_bust(data_dict, ['metadata', 'status']) res_id = _get_or_bust(metadata, 'resource_id') # Pass metadata, not data_dict, as it contains the resource id needed # on the auth checks p.toolkit.check_access('xloader_submit', context, metadata) task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': res_id, 'task_type': 'xloader', 'key': 'xloader' }) task['state'] = status task['last_updated'] = str(datetime.datetime.utcnow()) task['error'] = data_dict.get('error') resubmit = False if status in ('complete', 'running_but_viewable'): # Create default views for resource if necessary (only the ones that # require data to be in the DataStore) resource_dict = p.toolkit.get_action('resource_show')(context, { 'id': res_id }) dataset_dict = p.toolkit.get_action('package_show')( context, { 'id': resource_dict['package_id'] }) for plugin in p.PluginImplementations(xloader_interfaces.IXloader): plugin.after_upload(context, resource_dict, dataset_dict) logic.get_action('resource_create_default_resource_views')( context, { 'resource': resource_dict, 'package': dataset_dict, 'create_datastore_views': True, }) # Check if the uploaded file has been modified in the meantime if (resource_dict.get('last_modified') and metadata.get('task_created')): try: last_modified_datetime = parse_date( resource_dict['last_modified']) task_created_datetime = parse_date(metadata['task_created']) if last_modified_datetime > task_created_datetime: log.debug('Uploaded file more recent: {0} > {1}'.format( last_modified_datetime, task_created_datetime)) resubmit = True except ValueError: pass # Check if the URL of the file has been modified in the meantime elif (resource_dict.get('url') and metadata.get('original_url') and resource_dict['url'] != metadata['original_url']): log.debug('URLs are different: {0} != {1}'.format( resource_dict['url'], metadata['original_url'])) resubmit = True context['ignore_auth'] = True p.toolkit.get_action('task_status_update')(context, task) if resubmit: log.debug('Resource {0} has been modified, ' 'resubmitting to DataPusher'.format(res_id)) p.toolkit.get_action('xloader_submit')(context, { 'resource_id': res_id })
# idle cutoff for user inactivity IDLE_CUTOFF_DAYS = int(os.environ.get("IDLE_CUTOFF_DAYS") or 30) IDLE_CUTOFF = datetime.now(timezone.utc) - timedelta(days=IDLE_CUTOFF_DAYS) # limit the number of deletions in a given run IDLE_DELETE_LIMIT = int(os.environ.get("IDLE_DELETE_LIMIT") or 0) # batch variables for deletions DELETE_BATCH_SIZE = int(os.environ.get("DELETE_BATCH_SIZE") or 1) DELETE_BATCH_SECONDS = int(os.environ.get("DELETE_BATCH_SECONDS") or 30) # the date before which we assume sql data doesn't need to be deleted again # because re-running SQL delete is so slow # this should not normally be set, but it is now while we are re-importing data from the lake to the db SQL_CUTOFF_DATE = os.environ.get("SQL_CUTOFF_DATE") if SQL_CUTOFF_DATE: SQL_CUTOFF_DATE = parse_date(SQL_CUTOFF_DATE) # backlog date is a date cutoff so we can limit processing to # only old data when there's been a backlog buildup BACKLOG_DATE = os.environ.get("BACKLOG_DATE") PERSISTENT_CHECK_DB = os.environ.get("PERSISTENT_CHECK_DB", "") == "1" to_delete = graph.extension_attr_name("toDelete") to_delete_date = graph.extension_attr_name("toDeleteDate") iot_deleted_date = graph.extension_attr_name("iotDeletedDate") sql_deleted_date = graph.extension_attr_name("sqlDeletedDate") lake_deleted_date = graph.extension_attr_name("lakeDeletedDate") consent_revoked = graph.extension_attr_name("consentRevoked")
def step_impl(context, timestamp): context.cel['activation']["Now"] = celpy.celtypes.TimestampType(parse_date(timestamp))
def parse_sce_csv_file(path: str, service_id: str) -> List[IntervalReading]: """Extract interval data readings from a CSV file downloaded from the SCE website This file is little unusual; there is an extended header portion, several line containing various metadata about the interval data that follows. We skip over that header and extract the interval readings. They are returned as a list of tuples of the form (datetime.datetime, float). We expect these readings to be 15 minute readings. The readings can either be demand or usage values, depending on manner in which the CSV file is downloaded. This function makes no assumption about the units on these readings. A given file can contain reading for multiple service ids. This is implemented by having one column per service ID. This function takes an argument specifying which service id column to fetch data for. If the service ID cannot be found, an IntervalDataParseException is thrown. Similarly, any errors that occur while parsing will be raised as IntervalDataParseException instances. Arguments: path: The path to the CSV file on the file system service_id: The service id of interest Returns: A list of interval data readings, formatted as 2-tuples, of the for (datetime.datetime, float). The first tuple element stores the time when the reading occurred, the second the interval data reading, as a float. Raises: IntervalDataParseException: If the desired service ID can't be found, or an error occurs while parsing. """ # Read lines until we find the interval data header line (starts with the string "Date", # then store the raw data into data_lines data_started = False data_lines = [] with open(path) as f: for line in f: stripped_line = line.strip() if not data_started: if stripped_line.startswith("Date"): data_started = True data_lines.append(stripped_line) else: data_lines.append(stripped_line) # Parse each reading row csv_reader = csv.reader(data_lines) first = True data_column = None results = [] for row in csv_reader: if first: first = False headers = [th.strip() for th in row] for idx, th in enumerate(headers): if th == service_id: data_column = idx if not data_column: raise IntervalDataParseException( "Could not find data for SAID={}".format(service_id)) else: try: reading_date = parse_date(row[0].strip()).date() reading_time = parse_date(row[1].strip()).time() reading_datetime = datetime.combine(reading_date, reading_time) reading_value = _to_float(row[data_column].strip()) results.append( IntervalReading(dt=reading_datetime, value=reading_value)) except Exception as e: msg = "An error occured while trying to parse interval data from the SCE website." raise IntervalDataParseException(msg) from e return results
def datetime_to_date_string(datetime_obj): if isinstance(datetime_obj, str): datetime_obj = parse_date(datetime_obj) return "%s/%s/%s" % (datetime_obj.month, datetime_obj.day, datetime_obj.year)
def compute_features(self, threads, stemmed_vocabulary, distrib_matrix): self.log( 'Computing features. Please, wait. This will take some serious time...', logging.INFO) for thread in threads: self.log( 'Computing features for thread id {0}'.format( thread['question_uid']), logging.INFO) try: base_date = parse_date(thread['date_time']) except ValueError: base_date = parse_date('1970-01-01') except AttributeError: base_date = thread['date_time'] answers = thread['answers'] try: tag_list = thread['tags'].split('.') except AttributeError: tag_list = thread[ 'tags'] # there is no '.' used as tag separator if '' in tag_list: tag_list.remove('') for answer in answers: # compute thread tags answer_tags = answer['tags'].split() if 'null' in answer_tags: answer_tags.remove('null') tag_list.extend(answer_tags) thread['tags'] = sorted(set(tag_list)) # compute len in chars and words alen = len(answer['text']) answer['len'] = alen wordcount = Discretizer._count_words(answer['text']) answer['wordcount'] = wordcount if wordcount == 0: answer['avg_chars_per_word'] = 0 else: answer['avg_chars_per_word'] = "{0:.2f}".format( alen / float(wordcount)) # float with 2 decimals try: sentences = tokenize.sent_tokenize(answer['text'].decode( 'utf-8', 'replace').encode('ascii', 'replace'), language='english') except (AttributeError, TypeError) as e: sentences = tokenize.sent_tokenize(str(answer['text']), language='english') sentence_count = len(sentences) answer['sentences'] = sentence_count if sentence_count == 0: words_per_sentence = 0 else: words_per_sentence = "{0:.2f}".format( wordcount / float(sentence_count)) answer['avg_words_per_sentence'] = words_per_sentence longest_sentence = 0 for s in sentences: l = Discretizer._count_words(s) if l > longest_sentence: longest_sentence = l answer['longest_sentence'] = longest_sentence try: creation_date = parse_date(answer['date_time']) except AttributeError: creation_date = answer['date_time'] except Exception: print('\nInvalid date_time') time_difference = abs( (creation_date - base_date).total_seconds()) answer['time_difference'] = time_difference #answer['upvotes'] = thread['upvotes'] # check for urls and code snippets match = re.search(r'http(s)?://', str(answer['text']), re.MULTILINE) if match: answer['has_links'] = True else: answer['has_links'] = False answer['has_code_snippet'] = self._has_codesnippet( str(answer['text'])) try: LL = Discretizer._log_likelihood( answer['text'].decode('utf-8', 'replace').encode( 'ascii', 'replace'), stemmed_vocabulary, distrib_matrix) except (AttributeError, TypeError) as e: LL = Discretizer._log_likelihood(str(answer['text']), stemmed_vocabulary, distrib_matrix) answer['loglikelihood'] = LL answer['loglikelihood_descending'] = LL answer['loglikelihood_ascending'] = LL try: aspw = Discretizer._ASPW(answer['text'].decode( 'utf-8', 'replace').encode('ascii', 'replace')) except (AttributeError, TypeError) as e: aspw = Discretizer._ASPW(str(answer['text'])) fk = Discretizer._FK(answer['avg_words_per_sentence'], aspw) answer['F-K'] = fk answer['F-K_descending'] = fk answer['F-K_ascending'] = fk # compute ranks #answers = Discretizer._sort_rank(answers, 'upvotes', reverse=True) answers = Discretizer._sort_rank(answers, 'sentences', reverse=True) answers = Discretizer._sort_rank(answers, 'len', reverse=True) answers = Discretizer._sort_rank(answers, 'views', reverse=True) answers = Discretizer._sort_rank(answers, 'wordcount', reverse=True) answers = Discretizer._sort_rank(answers, 'avg_chars_per_word', reverse=True) answers = Discretizer._sort_rank(answers, 'avg_words_per_sentence', reverse=True) answers = Discretizer._sort_rank(answers, 'longest_sentence', reverse=True) answers = Discretizer._sort_rank(answers, 'time_difference', reverse=False) answers = Discretizer._sort_rank(answers, 'loglikelihood_descending', reverse=True) answers = Discretizer._sort_rank(answers, 'loglikelihood_ascending', reverse=False) answers = Discretizer._sort_rank(answers, 'F-K_descending', reverse=True) answers = Discretizer._sort_rank(answers, 'F-K_ascending', reverse=False) thread['answers'] = answers self.log( 'Done computing features for {0} threads'.format(len(threads)), logging.INFO) return threads
def _get_value(self, raw_value): return parse_date(raw_value)
def _prepare_attributes(name, value): name = name.lower() if name == 'site': value = value.lower() query = '' if value == 'acfun': query = 'acfun' elif value in ['bilibili', 'bili']: query = 'bilibili' elif value in ['youtube', 'ytb']: query = 'youtube' elif value in ['nicovideo', 'niconico', 'nico']: query = 'nicovideo' elif value in ['twitter']: query = 'twitter' elif value in ['ipfs']: query = 'ipfs' else: query = value return {'item.site': query} elif name == 'date': if value[:2] == '<=': date = parse_date(value[2:]) return {'item.upload_time': {'$lte': date + timedelta(days=1)}} elif value[:2] == '>=': date = parse_date(value[2:]) return {'item.upload_time': {'$gte': date}} elif value[:1] == '<': date = parse_date(value[1:]) return {'item.upload_time': {'$lt': date}} elif value[:1] == '>': date = parse_date(value[1:]) return {'item.upload_time': {'$gt': date}} elif value[:1] == '=': date = parse_date(value[1:]) return { 'item.upload_time': { '$gte': date, '$lte': date + timedelta(days=1) } } date = parse_date(value) return { 'item.upload_time': { '$gte': date, '$lte': date + timedelta(days=1) } } elif name == 'tags': if value[:2] == '<=': return {'tag_count': {'$lte': int(value[2:])}} elif value[:2] == '>=': return {'tag_count': {'$gte': int(value[2:])}} elif value[:1] == '<': return {'tag_count': {'$lt': int(value[1:])}} elif value[:1] == '>': return {'tag_count': {'$gt': int(value[1:])}} elif value[:1] == '=': return {'tag_count': {'$eq': int(value[1:])}} else: return {} elif name == 'placeholder': value = value.lower() if value == 'true': return {'item.placeholder': True} elif value == 'false': return {'item.placeholder': False} else: return {} elif name == 'repost': if value in _REPOST_TRANSLATE: value = _REPOST_TRANSLATE[value] if not value: return {'item.repost_type': {'$exists': False}} return {'item.repost_type': value} return {}
def test_publish_revision(content_util, persist_util, app, db_engines, db_tables): resources = list([content_util.gen_resource() for x in range(0, 2)]) module = content_util.gen_module(resources=resources) module = persist_util.insert_module(module) metadata = parse_module_metadata(module) # Collect control data for non-legacy metadata stmt = (db_tables.modules.select().where( db_tables.modules.c.moduleid == metadata.id)) control_metadata = db_engines['common'].execute(stmt).fetchone() # TARGET with pytest.raises(Unchanged), db_engines['common'].begin() as conn: now = conn.execute('SELECT CURRENT_TIMESTAMP as now').fetchone().now (id, version), ident = publish_legacy_page( module, metadata, ( 'user1', 'test publish', ), conn, ) # Change the module text, to make it publishable. index_cnxml = module.file.read_text() start_offset = index_cnxml.find('test document') module.file.write_text(index_cnxml[:start_offset] + 'TEST DOCUMENT' + index_cnxml[start_offset + 13:]) # TARGET - again with db_engines['common'].begin() as conn: now = conn.execute('SELECT CURRENT_TIMESTAMP as now').fetchone().now (id, version), ident = publish_legacy_page( module, metadata, ( 'user1', 'test publish', ), conn, ) # Check core metadata insertion stmt = (db_tables.modules.select().where( db_tables.modules.c.module_ident == ident)) result = db_engines['common'].execute(stmt).fetchone() assert result.version == '1.2' assert result.uuid == control_metadata.uuid assert result.major_version == 2 assert result.minor_version is None # Check for reuse of the existing abstract assert result.abstractid == control_metadata.abstractid assert result.created == parse_date(metadata.created) assert result.revised == now assert result.portal_type == 'Module' assert result.name == metadata.title assert result.licenseid == 13 assert result.submitter == 'user1' assert result.submitlog == 'test publish' assert result.authors == list(metadata.authors) assert result.maintainers == list(metadata.maintainers) assert result.licensors == list(metadata.licensors) assert result.google_analytics == GOOGLE_ANALYTICS_CODE # Check subject metadata insertion stmt = (db_tables.moduletags.join(db_tables.tags).select().where( db_tables.moduletags.c.module_ident == ident)) results = db_engines['common'].execute(stmt) subjects = [x.tag for x in results] assert sorted(subjects) == sorted(metadata.subjects) # Check keyword metadata insertion stmt = (db_tables.modulekeywords.join(db_tables.keywords).select().where( db_tables.modulekeywords.c.module_ident == ident)) results = db_engines['common'].execute(stmt) keywords = [x.word for x in results] assert sorted(keywords) == sorted(metadata.keywords) # Check for file insertion stmt = (db_tables.module_files.join(db_tables.files).select().where( db_tables.module_files.c.module_ident == ident)) result = db_engines['common'].execute(stmt).fetchall() files = {x.filename: x for x in result} assert len(files) == len(resources) + 2 # content files assert 'index.cnxml' in files assert 'index.cnxml.html' in files # Check for resource file insertion html_content = files['index.cnxml.html'].file.decode('utf8') for resource in resources: assert resource.filename in files # Check for reference rewrites in the content. This is out of scope # for this project, but order of insertion matters in order for # the references to be rewritten. assert '/resources/{}'.format(resource.sha1) in html_content
async def process_one(uuid_activity): uuid, last_activity = uuid_activity group = await graph.get_group(uuid) if group is None: app_log.info(f"No group for inactive device {uuid}") counts["no_group"] += 1 return if group.get(consent_revoked): app_log.info(f"Already marked for deletion: {uuid}") counts["deleted"] += 1 return user = await graph.user_for_device(group) if user is None: app_log.info(f"No user for inactive device {uuid}") counts["no_user"] += 1 # FIXME: something went wrong. Mark device id group for deletion? return # check other devices on the same user # in case of new device registrations, # don't delete data from a user's old phone other_device_activity = None device_ids = [uuid] async for group in graph.device_groups_for_user(user): device_id = group["displayName"] if device_id != uuid: device_ids.append(device_id) # First check for recent registration (cheap) if parse_date(group["createdDateTime"]) >= IDLE_CUTOFF: app_log.info(f"Recently registered device {device_id}") counts["new"] += 1 if device_id == uuid: app_log.warning( f"WRONG activity: recently registered {device_id} not idle" ) counts["wrong"] += 1 other_device_activity = True break try: device = await devices.get_device(device_id) except Exception as e: app_log.warning(f"Failed to get device {device_id}: ({e})") counts["iot_err"] += 1 pass else: if parse_date(device["lastActivityTime"]) >= IDLE_CUTOFF: counts["iot"] += 1 app_log.info(f"Activity on {device_id} in IoTHub") if device_id == uuid: app_log.warning( f"WRONG activity: iothub active {device_id} not idle" ) counts["wrong"] += 1 other_device_activity = True break if device_id != uuid: # if not registered since cutoff, check for activity in SQL if await check_sql_data(device_id, activity_cutoff=IDLE_CUTOFF): counts["sql"] += 1 app_log.info(f"Activity on {device_id} in SQL") other_device_activity = True break if other_device_activity: app_log.info(f"{uuid} is associated with other more recent device activity") counts["active"] += 1 else: app_log.info(f"User {user['logName']} is inactive since {last_activity}") app_log.info(f"Inactive devices: {','.join(device_ids)}") counts["idle"] += 1 return user
def __init__(self, item): self.guid = item.find('guid').text self.title = item.find('title').text self.link = item.find('link').text self.description = item.find('description').text self.pub_date = parse_date(item.find('pubDate').text)
def license_from_manifest(self, manifest): def is_appropriate_manifest_sub(sub): if sub['pool']['activeSubscription'] is False: return False now = datetime.now(timezone.utc) if parse_date(sub['startDate']) > now: return False if parse_date(sub['endDate']) < now: return False products = sub['pool']['providedProducts'] if any(product.get('productId') == '480' for product in products): return True return False def _can_aggregate(sub, license): # We aggregate multiple subs into a larger meta-sub, if they match # # No current sub in aggregate if not license: return True # Same SKU type (SER vs MCT vs others)? if license['sku'][0:3] != sub['pool']['productId'][0:3]: return False return True # Parse output for subscription metadata to build config license = dict() for sub in manifest: if not is_appropriate_manifest_sub(sub): logger.warning( "Subscription %s (%s) in manifest is not active or for another product" % (sub['pool']['productName'], sub['pool']['productId'])) continue if not _can_aggregate(sub, license): logger.warning( "Subscription %s (%s) in manifest does not match other manifest subscriptions" % (sub['pool']['productName'], sub['pool']['productId'])) continue license.setdefault('sku', sub['pool']['productId']) license.setdefault('subscription_name', sub['pool']['productName']) license.setdefault('pool_id', sub['pool']['id']) license.setdefault('product_name', sub['pool']['productName']) license.setdefault('valid_key', True) license.setdefault('license_type', 'enterprise') license.setdefault('satellite', False) # Use the nearest end date endDate = parse_date(sub['endDate']) currentEndDateStr = license.get('license_date', '4102462800') # 2100-01-01 currentEndDate = datetime.fromtimestamp(int(currentEndDateStr), timezone.utc) if endDate < currentEndDate: license['license_date'] = endDate.strftime('%s') instances = sub['quantity'] license['instance_count'] = license.get('instance_count', 0) + instances license['subscription_name'] = re.sub( r'[\d]* Managed Nodes', '%d Managed Nodes' % license['instance_count'], license['subscription_name']) if not license: logger.error("No valid subscriptions found in manifest") self._attrs.update(license) settings.LICENSE = self._attrs return self._attrs
def _prepare_attributes(name, value): value = value.lower() name = name.lower() if name == 'site': query = '' if value == 'acfun': query = 'acfun' elif value in ['bilibili', 'bili']: query = 'bilibili' elif value in ['youtube', 'ytb']: query = 'youtube' elif value in ['nicovideo', 'niconico', 'nico']: query = 'nicovideo' elif value in ['twitter']: query = 'twitter' elif value in ['ipfs']: query = 'ipfs' return {'item.site': query} elif name == 'date': if value[:2] == '<=': date = parse_date(value[2:]) return {'item.upload_time': {'$lte': date + timedelta(days=1)}} elif value[:2] == '>=': date = parse_date(value[2:]) return {'item.upload_time': {'$gte': date}} elif value[:1] == '<': date = parse_date(value[1:]) return {'item.upload_time': {'$lt': date}} elif value[:1] == '>': date = parse_date(value[1:]) return {'item.upload_time': {'$gt': date}} elif value[:1] == '=': date = parse_date(value[1:]) return { 'item.upload_time': { '$gte': date, '$lte': date + timedelta(days=1) } } date = parse_date(value) return { 'item.upload_time': { '$gte': date, '$lte': date + timedelta(days=1) } } elif name == 'tagless': if value == 'true': return {'tags': {'$size': 0}} elif value == 'false': return {'tags': {'$not': {'$size': 0}}} else: return {} elif name == 'placeholder': if value == 'true': return {'item.placeholder': True} elif value == 'false': return {'item.placeholder': False} else: return {} return {}
def _deserialize(self, value, attr, obj, **kwargs): try: return parse_date(value).date() except Exception as e: return None
def parse_prometheus(alert, external_url): status = alert.get('status', 'firing') labels = copy(alert['labels']) annotations = copy(alert['annotations']) starts_at = parse_date(alert['startsAt']) if alert['endsAt'] == '0001-01-01T00:00:00Z': ends_at = None else: ends_at = parse_date(alert['endsAt']) if status == 'firing': severity = labels.pop('severity', 'warning') create_time = starts_at elif status == 'resolved': severity = 'normal' create_time = ends_at else: severity = 'unknown' create_time = ends_at or starts_at # get labels resource = labels.pop('exported_instance', None) or labels.pop( 'instance', 'n/a') event = labels.pop('alertname') environment = labels.pop('environment', 'Production') # get annotations correlate = annotations.pop('correlate').split( ',') if 'correlate' in annotations else None service = annotations.pop('service', '').split(',') group = annotations.pop('job', 'Prometheus') value = annotations.pop('value', None) # build alert text summary = annotations.pop('summary', None) description = annotations.pop('description', None) text = description or summary or '%s: %s on %s' % ( labels['job'], labels['alertname'], labels['instance']) try: timeout = int(labels.pop('timeout', 0)) or None except ValueError: timeout = None if external_url: annotations['externalUrl'] = external_url if 'generatorURL' in alert: annotations[ 'moreInfo'] = '<a href="%s" target="_blank">Prometheus Graph</a>' % alert[ 'generatorURL'] return Alert( resource=resource, event=event, environment=environment, severity=severity, correlate=correlate, service=service, group=group, value=value, text=text, attributes=annotations, origin='prometheus/' + labels.pop('monitor', '-'), event_type='prometheusAlert', create_time=create_time.astimezone(tz=pytz.UTC).replace(tzinfo=None), timeout=timeout, raw_data=alert, tags=["%s=%s" % t for t in labels.items()] # any labels left are used for tags )
def validate_arg(arg, argdef): """ Validate an incoming (unicode) string argument according the UPnP spec. Raises UPNPError. """ datatype = argdef['datatype'] reasons = set() ranges = { 'ui1': (int, 0, 255), 'ui2': (int, 0, 65535), 'ui4': (int, 0, 4294967295), 'i1': (int, -128, 127), 'i2': (int, -32768, 32767), 'i4': (int, -2147483648, 2147483647), 'r4': (Decimal, Decimal('3.40282347E+38'), Decimal('1.17549435E-38')) } try: if datatype in set(ranges.keys()): v_type, v_min, v_max = ranges[datatype] if not v_min <= v_type(arg) <= v_max: reasons.add('%r datatype must be a number in the range %s to %s' % ( datatype, v_min, v_max)) elif datatype in {'r8', 'number', 'float', 'fixed.14.4'}: v = Decimal(arg) if v < 0: assert Decimal('-1.79769313486232E308') <= v <= Decimal('4.94065645841247E-324') else: assert Decimal('4.94065645841247E-324') <= v <= Decimal('1.79769313486232E308') elif datatype == 'char': v = arg.decode('utf8') if six.PY2 or isinstance(arg, bytes) else arg assert len(v) == 1 elif datatype == 'string': v = arg.decode("utf8") if six.PY2 or isinstance(arg, bytes) else arg if argdef['allowed_values'] and v not in argdef['allowed_values']: reasons.add('Value %r not in allowed values list' % arg) elif datatype == 'date': v = parse_date(arg) if any((v.hour, v.minute, v.second)): reasons.add("'date' datatype must not contain a time") elif datatype in ('dateTime', 'dateTime.tz'): v = parse_date(arg) if datatype == 'dateTime' and v.tzinfo is not None: reasons.add("'dateTime' datatype must not contain a timezone") elif datatype in ('time', 'time.tz'): now = datetime.datetime.utcnow() v = parse_date(arg, default=now) if v.tzinfo is not None: now += v.utcoffset() if not all(( v.day == now.day, v.month == now.month, v.year == now.year)): reasons.add('%r datatype must not contain a date' % datatype) if datatype == 'time' and v.tzinfo is not None: reasons.add('%r datatype must not have timezone information' % datatype) elif datatype == 'boolean': valid = {'true', 'yes', '1', 'false', 'no', '0'} if arg.lower() not in valid: reasons.add('%r datatype must be one of %s' % (datatype, ','.join(valid))) elif datatype == 'bin.base64': b64decode(arg) elif datatype == 'bin.hex': unhexlify(arg) elif datatype == 'uri': urlparse(arg) elif datatype == 'uuid': if not re.match( r'^[0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{12}$', arg, re.I): reasons.add('%r datatype must contain a valid UUID') else: reasons.add("%r datatype is unrecognised." % datatype) except ValueError as exc: reasons.add(str(exc)) return not bool(len(reasons)), reasons
def process(self, resources, event=None): results = [] filtered = [] attached = [] stats = Counter() marker_date = parse_date('2016-11-01T00:00:00+00:00') # Filter volumes for r in resources: # unsupported type if r['VolumeType'] == 'standard': stats['vol-type'] += 1 filtered.append(r['VolumeId']) continue # unattached are easy if not r.get('Attachments'): results.append(r) continue # check for attachment date older then supported date if r['Attachments'][0]['AttachTime'] < marker_date: stats['attach-time'] += 1 filtered.append(r['VolumeId']) continue attached.append(r) # Filter volumes attached to unsupported instance types ec2 = self.manager.get_resource_manager('ec2') instance_map = {} for v in attached: instance_map.setdefault(v['Attachments'][0]['InstanceId'], []).append(v) instances = ec2.get_resources(list(instance_map.keys())) for i in instances: if i['InstanceType'] in self.older_generation: stats['instance-type'] += len(instance_map[i['InstanceId']]) filtered.extend( [v['VolumeId'] for v in instance_map.pop(i['InstanceId'])]) else: results.extend(instance_map.pop(i['InstanceId'])) # Filter volumes that are currently under modification client = local_session(self.manager.session_factory).client('ec2') modifying = set() for vol_set in chunks(list(results), 200): vol_ids = [v['VolumeId'] for v in vol_set] mutating = client.describe_volumes_modifications( Filters=[{ 'Name': 'volume-id', 'Values': vol_ids }, { 'Name': 'modification-state', 'Values': ['modifying', 'optimizing', 'failed'] }]) for vm in mutating.get('VolumesModifications', ()): stats['vol-mutation'] += 1 filtered.append(vm['VolumeId']) modifying.add(vm['VolumeId']) self.log.debug("filtered %d of %d volumes due to %s", len(filtered), len(resources), sorted(stats.items())) return [r for r in results if r['VolumeId'] not in modifying]
def add_second(t): added = parse_date(t) + relativedelta(seconds=1) return added.isoformat()
def apptsuccess(request): flag = True p_name = request.user.id p_user = User.objects.get(id=p_name) d_name = request.POST.get("Doctor_name", "") d_user = User.objects.get(id=d_name) medProblem = request.POST.get("Medical_problem", "") date_unicode = request.POST.get("date", "") time_unicode = request.POST.get("time1", "") if (p_name != None and p_name != "" and d_name != None and d_name != '' and medProblem != None and medProblem != '' and date_unicode != None and date_unicode != '' and time_unicode != None and time_unicode != "" and parse_date(date_unicode) > datetime.now()): time_list = [] db_date_list = [] timedelta_list = [] date_list = [] date = datetime.strptime(date_unicode, "%Y-%m-%d") time = datetime.strptime(time_unicode, "%H:%M") calendar_date = datetime.strptime(date_unicode + " " + time_unicode, "%Y-%m-%d %H:%M") for a in User.objects.raw('SELECT * FROM login_appt;'): if (a.doctor_id == long(d_name)): time_list.append(a.time) date_list.append(a.date) db_date_list.append(datetime.combine(a.date, a.time)) if (calendar_date in db_date_list): flag = False else: for d in db_date_list: time_added = d + timedelta(minutes=15) if (calendar_date < time_added and calendar_date > d): flag = False break else: flag = True continue if flag == False: doctor_list = [] for doctor in User.objects.raw( 'SELECT * FROM auth_user a join login_userrole b on a.id=b.user_id where b.role="doctor";' ): doctor_list.append(doctor) return render_to_response( 'patient/appointment.html', { 'user': request.user, 'doctor': doctor_list, 'message': 'Sorry, There is a time clash, please select a different time.' }) else: d = Appt.objects.create(patient=p_user, doctor=d_user, medical_problem=medProblem, date=date, time=time) d.save() return render_to_response('patient/success.html', {'user': request.user}) else: doctor_list = [] for doctor in User.objects.raw( 'SELECT * FROM auth_user a join login_userrole b on a.id=b.user_id where b.role="doctor";' ): doctor_list.append(doctor) return render_to_response( 'patient/appointment.html', { 'user': request.user, 'doctor': doctor_list, 'message': 'Either the information you have filled is incorrect or empty. Please check.' }) d = Appt.objects.create(patient=p_user, doctor=d_user, medical_problem=medProblem, date=date, time=time) d.save() return render_to_response('patient/success.html', {'user': request.user})
def api_list_report_files(): probe_cc = request.args.get("probe_cc") probe_asn = request.args.get("probe_asn") test_name = request.args.get("test_name") since = request.args.get("since") until = request.args.get("until") since_index = request.args.get("since_index") order_by = request.args.get("order_by", "index") if order_by is "index": order_by = "idx" order = request.args.get("order", 'desc') try: offset = int(request.args.get("offset", "0")) limit = int(request.args.get("limit", "100")) except ValueError: raise BadRequest("Invalid offset or limit") q = current_app.db_session.query(ReportFile.filename, ReportFile.bucket_date, ReportFile.test_start_time, ReportFile.probe_cc, ReportFile.probe_asn, ReportFile.idx) # XXX maybe all of this can go into some sort of function. if probe_cc: q = q.filter(ReportFile.probe_cc == probe_cc) if probe_asn: q = q.filter(ReportFile.probe_asn == probe_asn) if test_name: q = q.filter(ReportFile.test_name == test_name) if since: try: since = parse_date(since) except ValueError: raise BadRequest("Invalid since") q = q.filter(ReportFile.test_start_time > since) if until: try: until = parse_date(until) except ValueError: raise BadRequest("Invalid until") q = q.filter(ReportFile.test_start_time <= until) if since_index: q = q.filter(ReportFile.idx > since_index) # XXX these are duplicated above, refactor into function if order.lower() not in ('asc', 'desc'): raise BadRequest("Invalid order") if order_by not in ('test_start_time', 'probe_cc', 'report_id', 'test_name', 'probe_asn', 'idx'): raise BadRequest("Invalid order_by") q = q.order_by('{} {}'.format(order_by, order)) count = q.count() pages = math.ceil(count / limit) current_page = math.ceil(offset / limit) + 1 q = q.limit(limit).offset(offset) next_args = request.args.to_dict() next_args['offset'] = "%s" % (offset + limit) next_args['limit'] = "%s" % limit next_url = urljoin(current_app.config['BASE_URL'], '/api/v1/files?%s' % urlencode(next_args)) if current_page >= pages: next_url = None metadata = { 'offset': offset, 'limit': limit, 'count': count, 'pages': pages, 'current_page': current_page, 'next_url': next_url } results = [] for row in q: url = get_download_url(current_app, row.bucket_date, row.filename) results.append({ 'download_url': url, 'probe_cc': row.probe_cc, 'probe_asn': row.probe_asn, # Will we ever hit sys.maxint? 'index': int(row.idx), 'test_start_time': row.test_start_time.strftime('%Y-%m-%dT%H:%M:%SZ') }) return jsonify({'metadata': metadata, 'results': results})
def build_list(type, video, listing, response): ADDON = xbmcaddon.Addon() engine = ADDON.getSetting('engine') streamType = ADDON.getSetting('streamType') kodi_version = int(getKodiVersion()) # Get the plugin url __url__ = sys.argv[0] try: attrs = video['attributes'] alternateTitle = attrs.get('alternateId') if type == 'sport': # Pull start time from availability window availability = video.get('attributes', {}).get('availabilityWindows', []) if len(availability) > 0: av_window = availability[0] av_start = parse_date(av_window['playableStart']) av_start_local = av_start.astimezone(tz.tzlocal()) av_startstr = av_start_local.strftime("%H:%M") title = av_startstr + ' - ' + attrs.get('name') elif type == 'daily': # Pull start time from schedule start av_start = parse_date(attrs['scheduleStart']) av_start_local = av_start.astimezone(tz.tzlocal()) av_startstr = av_start_local.strftime("%H:%M") title = av_startstr + ' - ' + attrs.get('name') elif type == 'ontv': # Pull start time from schedule start av_start = parse_date(attrs['scheduleStart']) av_start_local = av_start.astimezone(tz.tzlocal()) channel = attrs.get('path') if 'eurosport-1' in channel: title = 'Eurosport 1: ' + attrs.get('name') if 'eurosport-2' in channel: title = 'Eurosport 2: ' + attrs.get('name') else: # Set the base title title = attrs.get('name') # Add chanel details for on-air shows if type == 'daily': if attrs.get('materialType') == 'LINEAR': channel = attrs.get('path') if 'eurosport-1' in channel: title = title + ' - (E1)' if 'eurosport-2' in channel: title = title + ' - (E2)' if attrs.get('broadcastType') == 'LIVE': title = title + ' (Live)' item = ListItem(title) # Get image and it's url images = video.get('relationships', {}).get('images', {}).get('data', []) if len(images) > 0: image_url = response.get_image_url(images[0]['id']) item.setArt({'thumb': image_url, 'icon': image_url}) # Set the premiered date if type == 'daily' or type == 'ontv': premiered = str(attrs.get('scheduleStart')[:10]) timestamp = attrs.get('scheduleStart') if type == 'sport': premiered = str(attrs.get('publishStart')[:10]) timestamp = attrs.get('publishStart') # Get the plot plot = attrs.get('description') if plot == '' or plot is None: plot = attrs.get('secondaryTitle') # Set the metadata if type == 'ondemand': labels = {'title': title, 'sorttitle': title} else: labels = { 'title': title, 'sorttitle': title, 'plot': plot, 'premiered': premiered, 'aired': premiered, 'dateadded': timestamp, 'mediatype': 'episode' } item.setInfo('video', labels) if type == 'ondemand': isPlayable = 'false' else: now = datetime.datetime.now(tz.tzutc()) if av_start_local > now: isPlayable = 'false' else: isPlayable = 'true' item.setProperty('IsPlayable', isPlayable) # Ondemand brings up a list of items to select, not play if type == 'ondemand': url = '{0}?action=Select sport&sport={1}'.format( __url__, alternateTitle) isfolder = True else: # Determine which stream to play # Initialise properties mimetype = '' inputstreamtype = '' manifesttype = '' inputstream = '' manifest = '' # Matrix or earlier? if kodi_version > 18: inputstream = 'inputstream' else: if engine == 'inputstream.adaptive': inputstream = 'inputstreamaddon' if engine == 'ffmpeg': inputstream = 'inputstreamclass' # Set manifest & inputstream type if engine == 'inputstream.adaptive': manifest = 'inputstream.adaptive.manifest_type' inputstreamtype = 'inputstream.adaptive' if engine == 'ffmpeg': manifest = 'inputstream.ffmpegdirect.manifest_type' inputstreamtype = 'inputstream.ffmpegdirect' item.setProperty('inputstream.ffmpegdirect.is_realtime_stream', 'true') # Override settings for HLS streams if streamType == 'hls': mimetype = 'application/x-mpegURL' manifesttype = 'hls' # Override settings for ISM streams if streamType == 'ism': mimetype = 'text/xml' manifesttype = 'ism' # Set properties item.setContentLookup(False) item.setProperty(inputstream, inputstreamtype) item.setProperty(manifest, manifesttype) if (len(mimetype) != 0): item.setMimeType(mimetype) # Extra properties for ism ondemand streams if type == 'ondemand' and streamType == 'ism': item.setProperty('inputstream.ffmpegdirect.stream_mode', 'catchup') item.setProperty('inputstream.ffmpegdirect.open_mode', 'ffmpeg') item.setProperty('inputstream.ffmpegdirect.playback_as_live', 'true') id = video.get('id') url = '{0}?action=play&id={1}'.format(__url__, id) isfolder = False # Add item to our listing listing.append((url, item, isfolder)) except: pass
def get_row_first_seen(row, default=None): first_seen = default if "first_seen" in row: first_seen = parse_date(row["first_seen"]).replace(tzinfo=pytz.utc) return first_seen
def date_or_none(date): try: return parse_date(date) except Exception as error: logger.exception(error) return None
def parse_prometheus(alert: JSON, external_url: str) -> Alert: status = alert.get('status', 'firing') # Allow labels and annotations to use python string formats that refer to # other labels eg. runbook = 'https://internal.myorg.net/wiki/alerts/{app}/{alertname}' # See https://github.com/prometheus/prometheus/issues/2818 labels = {} for k, v in alert['labels'].items(): try: labels[k] = v.format(**alert['labels']) except Exception: labels[k] = v annotations = {} for k, v in alert['annotations'].items(): try: annotations[k] = v.format(**labels) except Exception: annotations[k] = v starts_at = parse_date(alert['startsAt']) if alert['endsAt'] != '0001-01-01T00:00:00Z': ends_at = parse_date(alert['endsAt']) else: ends_at = None # type: ignore if status == 'firing': severity = labels.pop('severity', 'warning') create_time = starts_at elif status == 'resolved': severity = alarm_model.DEFAULT_NORMAL_SEVERITY create_time = ends_at else: severity = 'unknown' create_time = ends_at or starts_at # labels resource = labels.pop('exported_instance', None) or labels.pop( 'instance', 'n/a') event = labels.pop('event', None) or labels.pop('alertname') environment = labels.pop('environment', 'Production') customer = labels.pop('customer', None) correlate = labels.pop('correlate').split( ',') if 'correlate' in labels else None service = labels.pop('service', '').split(',') group = labels.pop('group', None) or labels.pop('job', 'Prometheus') origin = 'prometheus/' + labels.pop('monitor', '-') tags = ['{}={}'.format(k, v) for k, v in labels.items() ] # any labels left over are used for tags try: timeout = int(labels.pop('timeout', 0)) or None except ValueError: timeout = None value = annotations.pop('value', None) summary = annotations.pop('summary', None) description = annotations.pop('description', None) text = description or summary or '{}: {} is {}'.format( severity.upper(), resource, event) if external_url: annotations[ 'externalUrl'] = external_url # needed as raw URL for bi-directional integration if 'generatorURL' in alert: annotations[ 'moreInfo'] = '<a href="{}" target="_blank">Prometheus Graph</a>'.format( alert['generatorURL']) attributes = annotations # any annotations left over are used for attributes return Alert( resource=resource, event=event, environment=environment, customer=customer, severity=severity, correlate=correlate, service=service, group=group, value=value, text=text, attributes=attributes, origin=origin, event_type='prometheusAlert', create_time=create_time.astimezone(tz=pytz.UTC).replace(tzinfo=None), timeout=timeout, raw_data=alert, tags=tags)