def test_format_timestamp(self): # specific datetime now = datetime.datetime(2013, 4, 28, 10, 0, 0) self.assertEqual(utils.format_timestamp(now), "20130428 100000") # timedelta now -= datetime.timedelta(seconds=30) self.assertEqual(utils.format_timestamp(now), "20130428 095930") # string self.assertEqual(utils.format_timestamp("20130428 095930"), "20130428 095930")
def to_json(self): import json return json.dumps( self, default=lambda o: o.__dict__ if type(o) is not datetime else utils.format_timestamp(o))
def transform_metric_data(metric_data, task_cnt=3): if metric_data["scores"]: metric_data["top_tasks"] = CustomSorter.sort_scores(metric_data["scores"])[:task_cnt] if "timestamp" in metric_data: metric_data["timestamp_formatted"] = utils.format_timestamp(metric_data["timestamp"]) if "previous_timestamp" in metric_data: metric_data["time_diff"] = utils.timediff(metric_data["timestamp"], metric_data["previous_timestamp"]) return metric_data
def generate_river_obj(redis_client, river_name): """ Return a dict that matches the river.js spec. """ river_key = 'rivers:%s' % river_name river_updates = [cPickle.loads(update) for update in redis_client.lrange(river_key, 0, -1)] return { 'updatedFeeds': { 'updatedFeed': river_updates, }, 'metadata': { 'docs': 'http://riverjs.org/', 'whenGMT': format_timestamp(arrow.utcnow()), 'whenLocal': format_timestamp(arrow.utcnow().to('local')), 'version': '3', 'secs': '', }, }
def decode(socket): fixed = bytearray(6) socket.recv_into(fixed, flags=MSG_WAITALL) file_path_length = byte_utils.bytes_to_char(fixed, 0) file_is_directory = byte_utils.bytes_to_boolean(fixed, 1) file_last_modified = byte_utils.bytes_to_unsigned_int(fixed, 2) file_size = None if not file_is_directory: fixed = bytearray(4) socket.recv_into(fixed, flags=MSG_WAITALL) file_size = byte_utils.bytes_to_unsigned_int(fixed, 0) strings = bytearray(file_path_length) socket.recv_into(strings, flags=MSG_WAITALL) file_path = byte_utils.bytes_to_string(strings, file_path_length, 0) if utils.DEBUG_LEVEL >= 3: utils.log_message("DEBUG", "Decoded send file packet: ") utils.log_message("DEBUG", "File path length: " + str(file_path_length)) utils.log_message("DEBUG", "Is directory: " + str(file_is_directory)) utils.log_message( "DEBUG", "Last modified: " + str(utils.format_timestamp(file_last_modified))) utils.log_message("DEBUG", "File size: " + str(file_size)) utils.log_message("DEBUG", "File Path: " + str(file_path)) # parse file's contents to File().write() 1024 chunks if is not directory if not file_is_directory: chunk_size = min(SendFilePacket.CHUNK_SIZE, file_size) remaining = file_size file_wrapper = File() received_bytes_acc = 0 while remaining > 0: if utils.DEBUG_LEVEL >= 3: utils.log_message("DEBUG", "Chunk size: " + str(chunk_size)) chunk = bytearray(chunk_size) received_bytes = socket.recv_into(chunk, flags=MSG_WAITALL) received_bytes_acc += received_bytes file_wrapper.write(chunk) remaining -= received_bytes chunk_size = min(chunk_size, remaining) file_wrapper.close() if utils.DEBUG_LEVEL >= 1: utils.log_message( "DEBUG", "File size is " + str(file_size) + " and received bytes are " + str(received_bytes_acc)) utils.log_message( "DEBUG", "File is located in " + str(file_wrapper.get_path())) else: file_wrapper = Directory() packet = SendFilePacket( FileInfo(file_path, file_is_directory, file_last_modified, file_size, file_wrapper)) return packet
def pretty_print_result(cls, result, points): for rec in result: if len(points) == 0: print(rec) else: print(utils.format_timestamp(rec[PointEnum.ts])) for point in points: print("{point}: {value}".format(point=point, value=rec[point])) print(os.linesep)
def receive_object(send_file_packet): """Receives a send file packet, and processes it""" info = send_file_packet.file_info utils.log_message("INFO", "Receiving object: " + info.path) info.file_wrapper.move(os.path.join(self.directory.get_path(), info.path)) info.file_wrapper.set_timestamp(info.last_modified) if utils.DEBUG_LEVEL >= 3: utils.log_message("DEBUG", "Object has been moved to: " + str(info.file_wrapper.get_path())) utils.log_message("DEBUG", "Timestamp has been set to: " + str(utils.format_timestamp(info.file_wrapper.get_timestamp()))) return 0
def decode(socket): fixed = bytearray(6) socket.recv_into(fixed, flags=MSG_WAITALL) username_length = byte_utils.bytes_to_char(fixed, 0) directory_name_length = byte_utils.bytes_to_char(fixed, 1) files_count = byte_utils.bytes_to_unsigned_int(fixed, 2) dynamic = bytearray(username_length + directory_name_length) socket.recv_into(dynamic, flags=MSG_WAITALL) username = byte_utils.bytes_to_string(dynamic, username_length, 0) directory_name = byte_utils.bytes_to_string(dynamic, directory_name_length, username_length) if utils.DEBUG_LEVEL >= 3: utils.log_message("DEBUG", "Decoded login packet: ") utils.log_message("DEBUG", "Username length: " + str(username_length)) utils.log_message( "DEBUG", "Directory name length: " + str(directory_name_length)) utils.log_message("DEBUG", "Files count: " + str(files_count)) utils.log_message("DEBUG", "Username: "******"DEBUG", "Directory name: " + str(directory_name)) utils.log_message("DEBUG", "Files: ") # Parse all file info files = [] for count in range(files_count): if utils.DEBUG_LEVEL >= 2: utils.log_message( "DEBUG", "Waiting for file info " + str(count) + "/" + str(files_count)) fixed = bytearray(6) socket.recv_into(fixed, flags=MSG_WAITALL) file_path_length = byte_utils.bytes_to_char(fixed, 0) file_is_directory = byte_utils.bytes_to_boolean(fixed, 1) file_last_modified = byte_utils.bytes_to_unsigned_int(fixed, 2) strings = bytearray(file_path_length) socket.recv_into(strings, flags=MSG_WAITALL) file_path = byte_utils.bytes_to_string(strings, file_path_length, 0) if utils.DEBUG_LEVEL >= 3: utils.log_message("DEBUG", "File path length: " + str(file_path_length)) utils.log_message("DEBUG", "Is directory: " + str(file_is_directory)) utils.log_message( "DEBUG", "File timestamp: " + str(utils.format_timestamp(file_last_modified))) utils.log_message("DEBUG", "File path: " + str(file_path)) files.append( FileInfo(file_path, file_is_directory, file_last_modified)) packet = LoginPacket(username, directory_name, files) return packet
def newgroups(self, since): """Send a NEWGROUPS command. Get all newsgroups created after ``since``. ``since`` can be a datetime, timedelta, or pre-formatted string (e.g., 'yymmdd hhmmss'). If it's a timedelta, subtract from datetime.utcnow first. """ ts = utils.format_timestamp(since) self.last_response, groups = self._server.longcmd("NEWGROUPS %s" % ts) return make_group_result(groups)
def populate_feed_update(self, entry): obj = { 'id': str(self.redis_client.incr('id-generator')), 'pubDate': format_timestamp(self.entry_timestamp(entry)), } # If both <title> and <description> exist: # title -> <title> # body -> <description> if entry.get('title') and entry.get('description'): obj['title'] = self.clean_text(entry.get('title')) obj['body'] = self.clean_text(entry.get('description')) # Drop the body if it's just a duplicate of the title. if obj['title'] == obj['body']: obj['body'] == '' # If <description> exists but <title> doesn't: # title -> <description> # body -> '' # # See http://scripting.com/2014/04/07/howToDisplayTitlelessFeedItems.html # for an ad-hoc spec. elif not entry.get('title') and entry.get('description'): obj['title'] = self.clean_text(entry.get('description')) obj['body'] = '' # If neither of the above work but <title> exists: # title -> <title> # body -> '' # # A rare occurance -- just about everybody uses both <title> # and <description> and those with title-less feeds just use # <description> (in keeping with the RSS spec) -- but the # Nieman Journalism Lab's RSS feed [1] needs this conditional # so I assume it's not the only one out there. # # [1] http://www.niemanlab.org/feed/ elif entry.get('title'): obj['title'] = self.clean_text(entry.get('title')) obj['body'] = '' if entry.get('link'): obj['link'] = entry.get('link') if entry.get('comments'): obj['comments'] = entry.get('comments') return obj
def set_last_session_timedelta(self): # Display the last session and total timedelta for the selected project. project_name = self.ui.projects_combo_box.currentText() if len(project_name) == 0: # when a task is removed currentText() returns '' self.current_index = 0 self.ui.projects_combo_box.setCurrentIndex(self.current_index) project_name = self.ui.projects_combo_box.currentText() total = self.state.get_total_timedelta(project_name) timestamp, ongoing = self.state.get_last_session_timedelta(project_name) timestamp, font_size = format_timestamp(str(timestamp)) self.ui.label_time_spent_value.setText(timestamp) self.ui.label_time_spent_value.setStyleSheet(f"font-size: {font_size}px") self.ui.label_total.setText(f"Total: {total}") self.toggle_labels(ongoing)
def receive_object(send_file_packet): """Receives a send file packet, and processes it""" info = send_file_packet.file_info utils.log_message("INFO", "Receiving object: " + info.path) info.file_wrapper.move( os.path.join(self.directory.get_path(), info.path)) info.file_wrapper.set_timestamp(info.last_modified) if utils.DEBUG_LEVEL >= 3: utils.log_message( "DEBUG", "Object has been moved to: " + str(info.file_wrapper.get_path())) utils.log_message( "DEBUG", "Timestamp has been set to: " + str( utils.format_timestamp( info.file_wrapper.get_timestamp()))) return 0
def decode(socket): fixed = bytearray(6) socket.recv_into(fixed, flags=MSG_WAITALL) file_path_length = byte_utils.bytes_to_char(fixed, 0) file_is_directory = byte_utils.bytes_to_boolean(fixed, 1) file_last_modified = byte_utils.bytes_to_unsigned_int(fixed, 2) file_size = None if not file_is_directory: fixed = bytearray(4) socket.recv_into(fixed, flags=MSG_WAITALL) file_size = byte_utils.bytes_to_unsigned_int(fixed, 0) strings = bytearray(file_path_length) socket.recv_into(strings, flags=MSG_WAITALL) file_path = byte_utils.bytes_to_string(strings, file_path_length, 0) if utils.DEBUG_LEVEL >= 3: utils.log_message("DEBUG", "Decoded send file packet: ") utils.log_message("DEBUG", "File path length: " + str(file_path_length)) utils.log_message("DEBUG", "Is directory: " + str(file_is_directory)) utils.log_message("DEBUG", "Last modified: " + str(utils.format_timestamp(file_last_modified))) utils.log_message("DEBUG", "File size: " + str(file_size)) utils.log_message("DEBUG", "File Path: " + str(file_path)) # parse file's contents to File().write() 1024 chunks if is not directory if not file_is_directory: chunk_size = min(SendFilePacket.CHUNK_SIZE, file_size) remaining = file_size file_wrapper = File() received_bytes_acc = 0 while remaining > 0: if utils.DEBUG_LEVEL >= 3: utils.log_message("DEBUG", "Chunk size: " + str(chunk_size)) chunk = bytearray(chunk_size) received_bytes = socket.recv_into(chunk, flags=MSG_WAITALL) received_bytes_acc += received_bytes file_wrapper.write(chunk) remaining -= received_bytes chunk_size = min(chunk_size, remaining) file_wrapper.close() if utils.DEBUG_LEVEL >= 1: utils.log_message("DEBUG", "File size is " + str(file_size) + " and received bytes are " + str( received_bytes_acc)) utils.log_message("DEBUG", "File is located in " + str(file_wrapper.get_path())) else: file_wrapper = Directory() packet = SendFilePacket(FileInfo(file_path, file_is_directory, file_last_modified, file_size, file_wrapper)) return packet
def decode(socket): fixed = bytearray(6) socket.recv_into(fixed, flags=MSG_WAITALL) username_length = byte_utils.bytes_to_char(fixed, 0) directory_name_length = byte_utils.bytes_to_char(fixed, 1) files_count = byte_utils.bytes_to_unsigned_int(fixed, 2) dynamic = bytearray(username_length + directory_name_length) socket.recv_into(dynamic, flags=MSG_WAITALL) username = byte_utils.bytes_to_string(dynamic, username_length, 0) directory_name = byte_utils.bytes_to_string(dynamic, directory_name_length, username_length) if utils.DEBUG_LEVEL >= 3: utils.log_message("DEBUG", "Decoded login packet: ") utils.log_message("DEBUG", "Username length: " + str(username_length)) utils.log_message("DEBUG", "Directory name length: " + str(directory_name_length)) utils.log_message("DEBUG", "Files count: " + str(files_count)) utils.log_message("DEBUG", "Username: "******"DEBUG", "Directory name: " + str(directory_name)) utils.log_message("DEBUG", "Files: ") # Parse all file info files = [] for count in range(files_count): if utils.DEBUG_LEVEL >= 2: utils.log_message("DEBUG", "Waiting for file info " + str(count) + "/" + str(files_count)) fixed = bytearray(6) socket.recv_into(fixed, flags=MSG_WAITALL) file_path_length = byte_utils.bytes_to_char(fixed, 0) file_is_directory = byte_utils.bytes_to_boolean(fixed, 1) file_last_modified = byte_utils.bytes_to_unsigned_int(fixed, 2) strings = bytearray(file_path_length) socket.recv_into(strings, flags=MSG_WAITALL) file_path = byte_utils.bytes_to_string(strings, file_path_length, 0) if utils.DEBUG_LEVEL >= 3: utils.log_message("DEBUG", "File path length: " + str(file_path_length)) utils.log_message("DEBUG", "Is directory: " + str(file_is_directory)) utils.log_message("DEBUG", "File timestamp: " + str(utils.format_timestamp(file_last_modified))) utils.log_message("DEBUG", "File path: " + str(file_path)) files.append(FileInfo(file_path, file_is_directory, file_last_modified)) packet = LoginPacket(username, directory_name, files) return packet
def run(self): while True: feed_url = self.inbox.get() try: feed_content = self.request_feed(feed_url) except requests.exceptions.RequestException as ex: logger.exception('Failed to check %s' % feed_url) future = arrow.utcnow() + timedelta(seconds=60*60) fmt = format_timestamp(future.to('local')) logger.info('Next check for %s: %s (%d seconds)' % (feed_url, fmt, 60*60)) self.redis_client.zadd('next_check', feed_url, future.timestamp) else: try: feed_parsed = feedparser.parse(feed_content) except ValueError as ex: logger.exception('Failed to parse %s' % feed_url) break feed_key = '%s:entries' % feed_url new_feed = (self.redis_client.llen(feed_key) == 0) feed_updates = [] timestamps = [] for entry in feed_parsed.entries: # We must keep track of feed updates so they're only seen # once. Here's how that happens: # # Redis stores a list at `feed_key` that contains # the 1000 (by default) most recently seen feed # update fingerprints. See self.entry_fingerprint # for how the fingerprint is calculated. # # If the fingerprint hasn't been seen before, add it # to `feed_key`. # # If it has, this feed update has already been seen # so we can skip it. if self.new_entry(feed_key, entry): self.add_feed_entry(feed_key, entry) else: continue update = self.populate_feed_update(entry) feed_updates.append(update) timestamps.append(self.entry_timestamp(entry)) timestamp_key = '%s:timestamps' % feed_url # Add any new timestamps found during this check if timestamps: logger.info('%d new entries for %s' % (len(timestamps), feed_url)) new_timestamps = [obj.timestamp for obj in timestamps] self.redis_client.lpush(timestamp_key, *new_timestamps) self.redis_client.sort(timestamp_key, desc=True, store=timestamp_key) self.redis_client.ltrim(timestamp_key, 0, 99) else: logger.info('No new entries for %s' % feed_url) history = self.redis_client.lrange(timestamp_key, 0, 9 if timestamps else 8) if not timestamps: # See http://goo.gl/X6QhWN for why we do this history.insert(0, arrow.utcnow().timestamp) delta = self.average_update_interval(history) # Don't check more than once a minute if delta.seconds < 60: delta = timedelta(seconds=60) # Cap the next check at two hours. elif delta.seconds > (2*60*60): logger.debug('Randomly scheduling %s' % feed_url) delta = timedelta(seconds=random.uniform(60*60, 2*60*60)) future_update = arrow.utcnow() + delta fmt = format_timestamp(future_update.to('local')) logger.info('Next check for %s: %s (%d seconds)' % (feed_url, fmt, delta.seconds)) self.redis_client.zadd('next_check', feed_url, future_update.timestamp) # Keep --initial most recent updates if this is the # first time we've seen the feed if new_feed: feed_updates = feed_updates[:self.cli_args.initial] if feed_updates: river_update = { 'feedDescription': feed_parsed.feed.get('description', ''), 'feedTitle': feed_parsed.feed.get('title', ''), 'feedUrl': feed_url, 'item': feed_updates, 'websiteUrl': feed_parsed.feed.get('link', ''), 'whenLastUpdate': format_timestamp(arrow.utcnow()), } for river_name in self.redis_client.smembers('%s:rivers' % feed_url): river_key = 'rivers:%s' % river_name self.redis_client.lpush(river_key, cPickle.dumps(river_update)) self.redis_client.ltrim(river_key, 0, self.cli_args.entries - 1) self.redis_client.sadd('updated_rivers', river_name) firehose_key = 'rivers:firehose' self.redis_client.lpush(firehose_key, cPickle.dumps(river_update)) self.redis_client.ltrim(firehose_key, 0, self.cli_args.entries - 1) self.redis_client.sadd('updated_rivers', 'firehose') finally: self.inbox.task_done()