class GathererTest(unittest.TestCase): def test_expected_order(self): for chunk in self.tr_log.chunks: self._gather(chunk) self.assertEquals(1, len(self.gatherer.pieces())) def test_random_order(self): while len(self.tr_log.chunks) > 0: index = random.randint(0, len(self.tr_log.chunks)-1) chunk = self.tr_log.chunks[index] del self.tr_log.chunks[index] self._gather(chunk) self.assertEquals(1, len(self.gatherer.pieces())) def setUp(self): self.tr_log = TrLogReader(SESSION_FILENAME).get_log() #self.segments = Interpreter().interpret(self.tr_log.chunks, self.tr_log.files) self.gatherer = Gatherer() self.files = [File(f["offset"], f["length"]) for f in self.tr_log.files] def _gather(self, piece_info): f = self.files[piece_info["filenum"]] begin = piece_info["begin"] - f.offset end = piece_info["end"] - f.offset self.gatherer.add(Piece(begin, end, f))
def test_add_second_piece(self): gatherer = Gatherer() gatherer.add(Piece(10, 20)) gatherer.add(Piece(40, 50)) expected_pieces = [Piece(10, 20), Piece(40, 50)] self.assertEquals(expected_pieces, gatherer.pieces())
def sysinfo(interval=5, host='localhost', port=35367): """Takes display name, port and update interval as arguments.""" g = Gatherer() d = DisplayClient(host, port) def backlight_switch(signum, frame): d.backlight() date = strftime("Date: %a %b %d, %Y ") d.write(1,1,string="%s %s %s %s" % (g.hostname, g.system, g.system_release, g.machine)) d.write(1,26,string=date) d.write(1,15,string="Uptime: ") d.write(1,37,string="Time: ") d.write(1,48,string="Load: ") try: while True: signal(SIGUSR1, backlight_switch) g.refresh() if date != strftime("Date: %a %b %d, %Y "): date = strftime("Date: %a %b %d, %Y ") d.write(1,26,string=date) d.write(51,15,string="%s " % ' '.join(g.uptime.split('up')[1].split('user')[0].split()[:-1])[:-1]) d.write(35,37,string=strftime("%H:%M")) d.write(35,48,string=g.uptime.split('average:')[-1].strip()) sleep(interval) except KeyboardInterrupt: d.close()
class File: def __init__(self, length, visualizer): self.visualizer = visualizer self.arriving_chunks = OrderedDict() self.gatherer = Gatherer() self.radius = 50.0 self.x = random.uniform(self.radius, visualizer.width - self.radius*2) self.y = random.uniform(self.radius, visualizer.height - self.radius*2) def add_chunk(self, chunk): chunk.boid = Boid(self.get_departure_position(chunk), 10.0, 3.0) chunk.arrival_position = self.get_arrival_position(chunk) chunk.boid.arrive(chunk.arrival_position) chunk.arrived = False self.arriving_chunks[chunk.id] = chunk def stopped_playing(self, chunk_id): chunk = self.arriving_chunks[chunk_id] del self.arriving_chunks[chunk_id] self.gatherer.add(chunk) def get_departure_position(self, chunk): if chunk.pan < 0.5: x = 0 else: x = self.visualizer.width y = chunk.height * self.visualizer.height return Vector2d(x, y) def get_arrival_position(self, chunk): angle = 2 * math.pi * chunk.begin / chunk.file_length x = self.x + self.radius * math.cos(angle) y = self.y + self.radius * math.sin(angle) return Vector2d(x, y)
def process_medialist(self, medialist, alwaysnotify=False, stop_on_error=False): self.init_run(len(medialist) > 0) processed = {'tvshow': {}, 'movie': [], 'episode': []} artcount = 0 currentitem = 0 if medialist: gatherer = Gatherer(self.monitor, self.only_filesystem) for mediaitem in medialist: if not mediainfo.is_known_mediatype(mediaitem): continue log('Processing {0}'.format(mediaitem['label']), xbmc.LOGINFO) if self.visible: self.progress.update(currentitem * 100 // len(medialist), message=mediaitem['label']) currentitem += 1 self.add_additional_iteminfo(mediaitem) cleaned = mediaitem.get_artwork_updates(cleaner.clean_artwork(mediaitem), mediaitem['art']) if cleaned: add_art_to_library(mediaitem['mediatype'], mediaitem.get('seasons'), mediaitem['dbid'], cleaned) mediaitem['art'].update(cleaned) mediaitem['art'] = dict(item for item in mediaitem['art'].iteritems() if item[1]) forcedart, availableart, services_hit, error = gatherer.getartwork(mediaitem) if error: header = L(PROVIDER_ERROR_MESSAGE).format(error['providername']) xbmcgui.Dialog().notification(header, error['message'], xbmcgui.NOTIFICATION_WARNING) log('{0}\n{1}'.format(header, error['message'])) if stop_on_error: break else: add_processeditem(processed, mediaitem) for arttype, imagelist in availableart.iteritems(): self.sort_images(arttype, imagelist, mediaitem['file']) existingart = dict(mediaitem['art']) # Remove existing local artwork if it is no longer available localart = [(arttype, image['url']) for arttype, image in forcedart.iteritems() \ if not image['url'].startswith('http')] selectedart = dict((arttype, None) for arttype, url in existingart.iteritems() \ if not url.startswith('http') and (arttype, url) not in localart) selectedart.update((key, image['url']) for key, image in forcedart.iteritems()) selectedart = mediainfo.renumber_all_artwork(selectedart) existingart.update(selectedart) selectedart.update(self.get_top_missing_art(mediaitem['mediatype'], existingart, availableart, mediaitem.get('seasons'))) selectedart = mediainfo.get_artwork_updates(mediaitem['art'], selectedart) if selectedart: add_art_to_library(mediaitem['mediatype'], mediaitem.get('seasons'), mediaitem['dbid'], selectedart) artcount += len(selectedart) if not services_hit: if self.monitor.abortRequested(): break elif self.monitor.waitForAbort(THROTTLE_TIME): break self.finish_run() if alwaysnotify or artcount: notifycount(artcount) return processed
def __init__(self, length, visualizer): self.visualizer = visualizer self.arriving_chunks = OrderedDict() self.gatherer = Gatherer() self.radius = 50.0 self.x = random.uniform(self.radius, visualizer.width - self.radius*2) self.y = random.uniform(self.radius, visualizer.height - self.radius*2)
def __init__(self, *args): visualizer.File.__init__(self, *args) self.position = self.visualizer.position_for_new_file() self.gatherer = Gatherer() self.inner_radius = self.visualizer.scale(RADIUS - CIRCLE_THICKNESS) self.radius = self.visualizer.scale(RADIUS) self.velocity = Vector2d(0,0)
def scrape(): scraper = LinkedinScraper() scraper.login(os.environ['LINKEDIN_USER'], os.environ['LINKEDIN_PW']) gatherer = Gatherer(scraper) watchdog = Watchdog(gatherer) job_list = [ 'data intern', 'machine learning intern', 'software intern', 'python intern' ] location_list = [ 'Vancouver', 'New York', 'San Francisco Bay Area', 'Toronto', ] for job in job_list: for location in location_list: print('gathering for', job, location) watchdog.monitor_gather(job, location) for job in job_list: print('gathering for', job, 'Worldwide') watchdog.monitor_gather(job, 'Worldwide') return 'Done'
def __init__(self, args): visualizer.Visualizer.__init__(self, args, file_class=File, peer_class=Peer, segment_class=Segment) self.inner_x = WALL_X self.outer_x = WALL_X + STAIRS_WIDTH self.wall_rear_x = WALL_X - WALL_WIDTH self.wall_bottom = self.step_y(NUM_STEPS) - STEP_HEIGHT self.stairs_depth = self.step_z(NUM_STEPS) self.files = {} self.segments = {} self.gatherer = Gatherer() self._segments_split_at_step_boundaries = None self._dragging_orientation = False self._dragging_y_position = False self._set_camera_position(CAMERA_POSITION) self._set_camera_orientation(CAMERA_Y_ORIENTATION, CAMERA_X_ORIENTATION) self.enable_accum() self.enable_3d() if self.args.waveform: self.gathered_color_v = CURSOR_COLOR_V * GATHERED_OPACITY + STEPS_COLOR_V * (1 - GATHERED_OPACITY) self.gathered_color_h = CURSOR_COLOR_H * GATHERED_OPACITY + STEPS_COLOR_H * (1 - GATHERED_OPACITY) self.subscribe_to_waveform() else: self.gathered_color_v = GATHERED_COLOR_V self.gathered_color_h = GATHERED_COLOR_H self.subscribe_to_amp()
def __init__(self, preferences: Preferences, cache: Cache, **kwargs): Gtk.Window.__init__(self, **kwargs) self.preferences = preferences self.cache = cache self.news_store = NewsStore() # self.gatherer = StubGatherer(self.news_store) self.gatherer = Gatherer(self.news_store) self.connect_signals() self.prepare_appearance() self.css_provider = self.create_css() self.current_view = None self.switch_view(self.news_store, self.preferences)
def __init__(self, args): visualizer.Visualizer.__init__(self, args, file_class=File, peer_class=Peer, segment_class=Segment) self.subscribe_to_waveform() self.playing_segments = collections.OrderedDict() self.gatherer = Gatherer()
def reset(self): visualizer.Visualizer.reset(self) self.playing_segments = collections.OrderedDict() self.gatherer = Gatherer() if self._gathered_segments_layer: self._gathered_segments_layer.refresh() self._first_segment_received = False self._title_renderer = None
def __init__(self, *args): visualizer.File.__init__(self, *args) self.gatherer = Gatherer() self.y = self.visualizer.step_y(self.filenum+1) self.z1 = self.visualizer.step_z(self.filenum) self.z2 = self.visualizer.step_z(self.filenum+1) self.z = (self.z1 + self.z2) / 2 self.neighbour_y = self.visualizer.step_y(self.filenum+2) self.neighbour_z1 = self.visualizer.step_z(self.filenum+1)
class File(visualizer.File): def __init__(self, *args): visualizer.File.__init__(self, *args) self.gatherer = Gatherer() self.x_scope = DynamicScope() def add_segment(self, segment): self.x_scope.put(segment.begin) self.x_scope.put(segment.end) segment.pan = (self.x_scope.map(segment.begin) + self.x_scope.map(segment.end)) / 2 segment.departure_position = segment.peer_position() self.visualizer.playing_segment(segment, segment.pan) def render(self): self.x_scope.update() self.draw_gathered_segments() def draw_gathered_segments(self): for segment in self.gatherer.pieces(): segment.draw_gathered() def byte_to_step_position(self, byte): return float(byte) / self.length * NUM_STEPS def byte_to_wall_position(self, byte): step_position = self.byte_to_step_position(byte) step = int(step_position)+1 return Vector2d(self.visualizer.step_z(step_position), self.visualizer.step_y(step)) def byte_surfaces(self, begin, end, min_size=None): step_pos1 = self.byte_to_step_position(begin) step_pos2 = self.byte_to_step_position(end) if (step_pos2 - step_pos1) < min_size: step_pos1, step_pos2 = self.resize(step_pos1, step_pos2, min_size) step1 = int(step_pos1) step2 = int(step_pos2) fraction1 = step_pos1 % 1 fraction2 = step_pos2 % 1 for step in range(step1, step2+1): if step == step1: relative_begin = fraction1 else: relative_begin = 0 if step == step2: relative_end = fraction2 else: relative_end = 1 step_surfaces = list(self.visualizer.step_surfaces(step, relative_begin, relative_end)) yield step_surfaces[1] def resize(self, pos1, pos2, new_size): new_pos1 = max(pos1 - new_size/2, 0) new_pos2 = min(pos2 + new_size/2, NUM_STEPS) return new_pos1, new_pos2
def test_overlap_multiple_pieces(self): gatherer = Gatherer() gatherer.add(Piece(10, 20)) gatherer.add(Piece(30, 40)) gatherer.add(Piece(0, 50)) expected_pieces = [Piece(0, 50)] self.assertEquals(expected_pieces, gatherer.pieces())
def test_fit_hole(self): gatherer = Gatherer() gatherer.add(Piece(10, 20)) gatherer.add(Piece(40, 50)) gatherer.add(Piece(20, 40)) expected_pieces = [Piece(10, 50)] self.assertEquals(expected_pieces, gatherer.pieces())
def test_prepend(self): gatherer = Gatherer() gatherer.add(Piece(10, 20)) gatherer.add(Piece(40, 50)) gatherer.add(Piece(30, 40)) expected_pieces = [Piece(10, 20), Piece(30, 50)] self.assertEquals(expected_pieces, gatherer.pieces())
def test_multiple_files_append(self): f1 = File(offset=0, length=30) f2 = File(offset=30, length=20) gatherer = Gatherer() gatherer.add(Piece(20, 30, f1)) gatherer.add(Piece(0, 10, f2)) expected_pieces = [Piece(20, 40, f1)] self.assertEquals(expected_pieces, gatherer.pieces())
class File: def __init__(self, length, visualizer): self.visualizer = visualizer self.arriving_chunks = OrderedDict() self.gatherer = Gatherer() def add_chunk(self, chunk): self.arriving_chunks[chunk.id] = chunk chunk.setup() def update(self): self.update_arriving_chunks() def update_arriving_chunks(self): for chunk in self.arriving_chunks.values(): chunk.update() if chunk.arrived(): self.gather_chunk(chunk) def gather_chunk(self, chunk): del self.arriving_chunks[chunk.id] chunk.has_arrived = True self.gatherer.add(chunk)
def sysinfo(interval=5, host='localhost', port=35367): """Takes display name, port and update interval as arguments.""" g = Gatherer() d = DisplayClient(host, port) def backlight_switch(signum, frame): d.backlight() date = strftime("Date: %a %b %d, %Y ") d.write(1, 1, string="%s %s %s %s" % (g.hostname, g.system, g.system_release, g.machine)) d.write(1, 26, string=date) d.write(1, 15, string="Uptime: ") d.write(1, 37, string="Time: ") d.write(1, 48, string="Load: ") try: while True: signal(SIGUSR1, backlight_switch) g.refresh() if date != strftime("Date: %a %b %d, %Y "): date = strftime("Date: %a %b %d, %Y ") d.write(1, 26, string=date) d.write( 51, 15, string="%s " % ' '.join( g.uptime.split('up')[1].split('user')[0].split()[:-1])[:-1] ) d.write(35, 37, string=strftime("%H:%M")) d.write(35, 48, string=g.uptime.split('average:')[-1].strip()) sleep(interval) except KeyboardInterrupt: d.close()
def __init__(self, preferences, cache, **kwargs): Gtk.Window.__init__(self, **kwargs) # Signals self.connect('key_press_event', self.on_key_press) self.connect('item_scraped_event', self.on_item_scraped) self.connect('feed_gathered_event', self.on_feed_gathered) self.set_good_default_size() self.set_window_icon() self.preferences = preferences self.css_provider = self.create_css() self.header_bar = self.create_header() self.cache = cache self.gatherer = Gatherer(self, self.preferences, self.cache) self.current_view = None self.switch_view() self.gatherer.request_feeds()
class File(visualizer.File): def __init__(self, *args): visualizer.File.__init__(self, *args) self.gatherer = Gatherer() self.x_scope = DynamicScope() def add_segment(self, segment): self.x_scope.put(segment.begin) self.x_scope.put(segment.end) segment.pan = (self.x_scope.map(segment.begin) + self.x_scope.map(segment.end)) / 2 segment.departure_position = segment.peer_position() self.visualizer.playing_segment(segment) def render(self): self.x_scope.update() self.draw_gathered_segments() def draw_gathered_segments(self): for segment in self.gatherer.pieces(): segment.draw_gathered() def byte_to_coord(self, byte): return self.visualizer.prepend_margin_width + \ self.x_scope.map(byte) * self.visualizer.safe_width
class File(visualizer.File): def __init__(self, *args): visualizer.File.__init__(self, *args) self.gatherer = Gatherer() self.y = self.visualizer.step_y(self.filenum+1) self.z1 = self.visualizer.step_z(self.filenum) self.z2 = self.visualizer.step_z(self.filenum+1) self.z = (self.z1 + self.z2) / 2 self.neighbour_y = self.visualizer.step_y(self.filenum+2) self.neighbour_z1 = self.visualizer.step_z(self.filenum+1) def add_segment(self, segment): segment.departure_position = segment.peer.position self.visualizer.playing_segment(segment) def render(self): self.draw_gathered_segments() def draw_gathered_segments(self): for segment in self.gatherer.pieces(): segment.draw_gathered() def byte_to_x(self, byte): return WALL_X + float(byte) / self.length * STAIRS_WIDTH
def reset(self): visualizer.Visualizer.reset(self) self.playing_segments = collections.OrderedDict() self.gatherer = Gatherer()
def __init__(self, *args): visualizer.File.__init__(self, *args) self.playing_segments = OrderedDict() self.gatherer = Gatherer() self.x_scope = DynamicScope()
class Waves(visualizer.Visualizer): def __init__(self, args): visualizer.Visualizer.__init__(self, args, file_class=File, segment_class=Segment) self.subscribe_to_waveform() def reset(self): visualizer.Visualizer.reset(self) self.playing_segments = collections.OrderedDict() self.gatherer = Gatherer() def pan_segment(self, segment): # let orchestra & synth spatialize pass def InitGL(self): visualizer.Visualizer.InitGL(self) glClearColor(0.0, 0.0, 0.0, 0.0) self._gathered_segments_layer = self.new_layer(self._render_gathered_segments) def update(self): outdated = [] for segment in self.playing_segments.values(): if not segment.is_playing(): self.gatherer.add(segment) outdated.append(segment.id) if len(outdated) > 0: for segment_id in outdated: del self.playing_segments[segment_id] self._gathered_segments_layer.refresh() def render(self): self.update() self._gathered_segments_layer.draw() self.draw_playing_segments() def draw_playing_segments(self): glEnable(GL_LINE_SMOOTH) glHint(GL_LINE_SMOOTH_HINT, GL_NICEST) glEnable(GL_BLEND) glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA) for segment in self.playing_segments.values(): segment.render() def _render_gathered_segments(self): glBegin(GL_QUADS) x1 = 0 x2 = self.width min_height = GATHERED_LINE_WIDTH * self.height for segment in self.gatherer.pieces(): y1 = self.byte_to_py(segment.torrent_begin) y2 = max(self.byte_to_py(segment.torrent_end), y1 + min_height) if (y2 - y1) > min_height: d = min((y2 - y1) * 0.2, MAX_GRADIENT_HEIGHT * self.height) y1d = y1 + d y2d = y2 - d glColor3f(0, 0, 0) glVertex2f(x1, y1) glColor3f(*GATHERED_COLOR) glVertex2f(x1, y1d) glVertex2f(x2, y1d) glColor3f(0, 0, 0) glVertex2f(x2, y1) glColor3f(0, 0, 0) glVertex2f(x1, y2) glColor3f(*GATHERED_COLOR) glVertex2f(x1, y2d) glVertex2f(x2, y2d) glColor3f(0, 0, 0) glVertex2f(x2, y2) glColor3f(*GATHERED_COLOR) glVertex2f(x1, y1d) glVertex2f(x1, y2d) glVertex2f(x2, y2d) glVertex2f(x2, y1d) else: glColor3f(0, 0, 0) glVertex2f(x1, y1) glColor3f(*GATHERED_COLOR) glVertex2f(x1, y2) glVertex2f(x2, y2) glColor3f(0, 0, 0) glVertex2f(x2, y1) glEnd() def byte_to_py(self, byte): return int(self.byte_to_relative_y(byte) * self.height) def byte_to_relative_y(self, byte): return float(byte) / self.torrent_length def handle_segment_waveform_value(self, segment, value): segment.waveform.appendleft(value) def finished(self): if len(self.gatherer.pieces()) == 1: piece = self.gatherer.pieces()[0] return piece.begin == 0 and piece.end == self.torrent_length else: return False
class File: def __init__(self, length, visualizer): self.visualizer = visualizer self.arriving_chunks = OrderedDict() self.gatherer = Gatherer() def add_chunk(self, chunk): chunk.boid = Boid(self.get_departure_position(chunk), 10.0, 3.0) chunk.target = None chunk.target_type = None chunk.target_position = None chunk.target_position = self.get_target_position(chunk) chunk.boid.arrive(chunk.target_position) chunk.arrived = False self.arriving_chunks[chunk.id] = chunk def stopped_playing(self, chunk_id): chunk = self.arriving_chunks[chunk_id] self.gather_chunk(chunk) def gather_chunk(self, chunk): del self.arriving_chunks[chunk.id] # TODO: verify that these positions are really OK chunk.begin_position = Vector(chunk.boid.loc.x, chunk.boid.loc.y) chunk.end_position = Vector(chunk.boid.loc.x, chunk.boid.loc.y) self.gatherer.add(chunk) self.reorient_chunks_following(chunk) def get_departure_position(self, chunk): if chunk.pan < 0.5: x = 0 else: x = self.visualizer.width y = chunk.height * self.visualizer.height return Vector(x, y) def get_target_position(self, chunk): if not chunk.target_position: self.find_target(chunk) return chunk.target_position def find_target(self, chunk): position = self.find_joinable_piece(chunk) if position: chunk.target_type = TargetType.PIECE chunk.target_position = position return other_chunk = self.find_other_huntable_chunk(chunk) if other_chunk: chunk.target_type = TargetType.CHUNK chunk.target = other_chunk chunk.target_position = other_chunk.boid.loc return if len(self.gatherer.pieces()) > 0: chunk.target_position = self.close_to_existing_piece() return chunk.target_type = TargetType.NEW_PIECE chunk.target_position = self.anywhere() def anywhere(self): return Vector(random.uniform(self.visualizer.width * INNER_MARGIN, self.visualizer.width * (1 - INNER_MARGIN*2)), random.uniform(self.visualizer.height * INNER_MARGIN, self.visualizer.height * (1 - INNER_MARGIN*2))) def close_to_existing_piece(self): piece = random.choice(self.gatherer.pieces()) position = random.choice([piece.begin_position, piece.end_position]) angle = random.uniform(0, 2 * math.pi) distance = 1.0 movement = Vector(distance * math.cos(angle), distance * math.sin(angle)) return position + movement def find_joinable_piece(self, chunk): appendable_piece_key = self.gatherer.find_appendable_piece(chunk) prependable_piece_key = self.gatherer.find_prependable_piece(chunk) if appendable_piece_key and prependable_piece_key: appendable_position = self.gatherer.piece(appendable_piece_key).begin_position prepandable_position = self.gatherer.piece(prependable_piece_key).end_position chunk.target_position = (appendable_position + prepandable_position) / 2 elif appendable_piece_key: return self.gatherer.piece(appendable_piece_key).begin_position elif prependable_piece_key: return self.gatherer.piece(prependable_piece_key).end_position def find_other_huntable_chunk(self, chunk): # optimization: iterate appendables in reverse order, or use some kind of cache for other in self.arriving_chunks.values(): if other.end == chunk.begin: return other if other.begin == chunk.end: return other def reorient_chunks_following(self, targeted_chunk): for chunk in self.arriving_chunks.values(): if chunk.target_type == TargetType.CHUNK and \ chunk.target == targeted_chunk: self.find_target(chunk) def update(self): self.update_pieces() self.update_arriving_chunks() def update_pieces(self): for piece in self.gatherer.pieces(): self.get_piece_force(piece) for piece in self.gatherer.pieces(): self.apply_piece_force(piece) def get_piece_force(self, piece): piece.desired_length = self.desired_piece_length(piece) piece.begin_force = Vector(0,0) piece.end_force = Vector(0,0) self.consider_desired_length(piece, piece.begin_force, piece.begin_position, piece.end_position) self.consider_desired_length(piece, piece.end_force, piece.end_position, piece.begin_position) piece.begin_force.limit(3.0) piece.end_force.limit(3.0) def desired_piece_length(self, piece): return piece.byte_size / 10 def consider_desired_length(self, piece, force, position, opposite_position): force += spring_force(position, opposite_position, piece.desired_length) def apply_piece_force(self, piece): piece.begin_position += piece.begin_force piece.end_position += piece.end_force def update_arriving_chunks(self): for chunk in self.arriving_chunks.values(): if not chunk.arrived: chunk.boid.update() if self.arrived(chunk): #self.visualizer.play_chunk(chunk) # TEMP chunk.arrived = True self.gather_chunk(chunk) # TEMP def arrived(self, chunk): if chunk.target_type in [TargetType.PIECE, TargetType.NEW_PIECE]: distance = (chunk.target_position - chunk.boid.loc).mag() return distance < 1.0 else: return False
class File(visualizer.File): def __init__(self, *args): visualizer.File.__init__(self, *args) self.playing_segments = OrderedDict() self.gatherer = Gatherer() self.x_scope = DynamicScope() def add_segment(self, segment): self.x_scope.put(segment.begin) self.x_scope.put(segment.end) pan = (self.byte_to_coord(segment.begin) + self.byte_to_coord(segment.end)) / 2 self.visualizer.playing_segment(segment, pan) self.playing_segments[segment.id] = segment def update(self): outdated = filter(lambda segment_id: self.playing_segments[segment_id].relative_age() > 1, self.playing_segments) for segment_id in outdated: self.gatherer.add(self.playing_segments[segment_id]) del self.playing_segments[segment_id] self.x_scope.update() def render(self): self.y = self.visualizer.filenum_to_y_coord(self.filenum) self.draw_gathered_segments() self.draw_playing_segments() def draw_gathered_segments(self): for segment in self.gatherer.pieces(): self.draw_segment(segment, 0) def draw_playing_segments(self): for segment in self.playing_segments.values(): self.draw_playing_segment(segment) def draw_playing_segment(self, segment): actuality = 1 - segment.relative_age() self.draw_segment(segment, actuality) def draw_segment(self, segment, actuality): y_offset = actuality * 10 height = 3 + actuality * 10 y1 = int(self.y + y_offset) y2 = int(self.y + y_offset + height) x1 = self.visualizer.prepend_margin_width + \ int(self.byte_to_coord(segment.begin) * self.visualizer.safe_width) x2 = self.visualizer.prepend_margin_width + \ int(self.byte_to_coord(segment.end) * self.visualizer.safe_width) x1, x2 = self.upscale(x1, x2, actuality) if x2 == x1: x2 = x1 + 1 opacity = 0.2 + (actuality * 0.8) glColor3f(1-opacity, 1-opacity, 1-opacity) glBegin(GL_LINE_LOOP) glVertex2i(x1, y2) glVertex2i(x2, y2) glVertex2i(x2, y1) glVertex2i(x1, y1) glEnd() def upscale(self, x1, x2, actuality): unscaled_size = x2 - x1 desired_size = actuality * ARRIVAL_SIZE if desired_size > unscaled_size: mid = (x1 + x2) / 2 half_desired_size = int(desired_size/2) x1 = mid - half_desired_size x2 = mid + half_desired_size return (x1, x2) def byte_to_coord(self, byte): return self.x_scope.map(byte)
def test_overlap_begin(self): gatherer = Gatherer() gatherer.add(Piece(10, 20)) gatherer.add(Piece(0, 15)) expected_pieces = [Piece(0, 20)] self.assertEquals(expected_pieces, gatherer.pieces())
from pycallgraph import PyCallGraph from pycallgraph.output import GraphvizOutput from gatherer import Gatherer tickrate = 60 with PyCallGraph(output=GraphvizOutput()): aaron = Gatherer(startingpos=[380, 280]) aaron.update()
def __init__(self, length, visualizer): self.visualizer = visualizer self.arriving_chunks = OrderedDict() self.gatherer = Gatherer()
def monitor_gather(self, keywords, location, start=0): while True: #try: self.gatherer.gather_jobs(keywords, location, start, progress_file='progress') break #except Exception as e: # print('ERROR!') # print(e) # print('********************************') # start = int(open('progress').read()) if __name__ == "__main__": import os from gatherer import Gatherer from linkedinScraper import LinkedinScraper scraper = LinkedinScraper() #scraper.login(os.environ['LINKEDIN_USER'], os.environ['LINKEDIN_PW']) gatherer = Gatherer(scraper) watchdog = Watchdog(gatherer) keywords = 'software intern' location = 'Worldwide' watchdog.monitor_gather(keywords, location)
from linkedinScraper import LinkedinScraper from gatherer import Gatherer scraper = LinkedinScraper() scraper.login(input('Username?'), input('Password?')) gatherer = Gatherer(scraper) job_list = [ 'data intern', 'machine learning intern', 'software intern', 'python intern' ] location_list = [ 'Vancouver', 'New York', 'San Francisco Bay Area', 'Toronto' ] search_list = [] for job in job_list: for location in location_list: print('gathering for', job, location) gatherer.gather_jobs(job, location) gatherer.digest_data()
def process_item(self, mediatype, dbid, mode): if self.processor_busy: return if mode == MODE_GUI: xbmc.executebuiltin('ActivateWindow(busydialog)') if mediatype == mediatypes.TVSHOW: mediaitem = quickjson.get_tvshow_details(dbid, tvshow_properties) elif mediatype == mediatypes.MOVIE: mediaitem = quickjson.get_movie_details(dbid, movie_properties) elif mediatype == mediatypes.EPISODE: mediaitem = quickjson.get_episode_details(dbid, episode_properties) else: xbmc.executebuiltin('Dialog.Close(busydialog)') xbmcgui.Dialog().notification("Artwork Beef", L(NOT_SUPPORTED_MESSAGE).format(mediatype), '-', 6500) return if mode == MODE_GUI: self.init_run() self.add_additional_iteminfo(mediaitem) gatherer = Gatherer(self.monitor, self.only_filesystem) forcedart, availableart, _, error = gatherer.getartwork(mediaitem, False) if error: header = L(PROVIDER_ERROR_MESSAGE).format(error['providername']) xbmcgui.Dialog().notification(header, error['message'], xbmcgui.NOTIFICATION_WARNING) log('{0}\n{1}'.format(header, error['message'])) for arttype, imagelist in availableart.iteritems(): self.sort_images(arttype, imagelist, mediaitem['file']) xbmc.executebuiltin('Dialog.Close(busydialog)') if availableart: if 'seasons' in mediaitem and 'fanart' in availableart: for season in mediaitem['seasons'].keys(): unseasoned_backdrops = [dict(art) for art in availableart['fanart'] if not art.get('hasseason')] key = 'season.{0}.fanart'.format(season) if key in availableart: availableart[key].extend(unseasoned_backdrops) else: availableart[key] = unseasoned_backdrops tag_forcedandexisting_art(availableart, forcedart, mediaitem['art']) selectedarttype, selectedart = prompt_for_artwork(mediaitem['mediatype'], mediaitem['label'], availableart, self.monitor) if selectedarttype: if selectedarttype.startswith('season.'): gentype = selectedarttype.rsplit('.', 1)[1] multiselect = mediatypes.artinfo[mediatypes.SEASON].get(gentype, {}).get('multiselect', False) else: multiselect = mediatypes.artinfo[mediaitem['mediatype']][selectedarttype]['multiselect'] if multiselect: existingurls = [url for exacttype, url in mediaitem['art'].iteritems() \ if mediainfo.arttype_matches_base(exacttype, selectedarttype)] urls_toset = [url for url in existingurls if url not in selectedart[1]] urls_toset.extend([url for url in selectedart[0] if url not in urls_toset]) selectedart = dict(mediainfo.iter_renumbered_artlist(selectedart, selectedarttype, mediaitem['art'].keys())) else: selectedart = {selectedarttype: selectedart} selectedart = mediainfo.get_artwork_updates(mediaitem['art'], selectedart) if selectedart: add_art_to_library(mediaitem['mediatype'], mediaitem.get('seasons'), mediaitem['dbid'], selectedart) notifycount(len(selectedart)) else: xbmcgui.Dialog().notification(L(NOT_AVAILABLE_MESSAGE), L(SOMETHING_MISSING) + ' ' + L(FINAL_MESSAGE), '-', 8000) self.finish_run() else: medialist = [mediaitem] autoaddepisodes = addon.get_setting('autoaddepisodes_list') if addon.get_setting('episode.fanart') else () if mediatype == mediatypes.TVSHOW and mediaitem['imdbnumber'] in autoaddepisodes: medialist.extend(quickjson.get_episodes(dbid, properties=episode_properties)) self.process_medialist(medialist, True)
class MainWindow(Gtk.Window): __gtype_name__ = 'TroughWindow' def __init__(self, preferences: Preferences, cache: Cache, **kwargs): Gtk.Window.__init__(self, **kwargs) self.preferences = preferences self.cache = cache self.news_store = NewsStore() # self.gatherer = StubGatherer(self.news_store) self.gatherer = Gatherer(self.news_store) self.connect_signals() self.prepare_appearance() self.css_provider = self.create_css() self.current_view = None self.switch_view(self.news_store, self.preferences) def connect_signals(self) -> None: self.connect('key_press_event', self.on_key_press) def prepare_appearance(self) -> None: self.set_good_default_size() self.set_window_icon() self.create_header() def set_good_default_size(self) -> None: screen = self.get_screen() active_window = screen.get_active_window() if active_window: monitor = screen.get_monitor_at_window(active_window) geometry = screen.get_monitor_geometry(monitor) width = floor(.60 * geometry.width) height = floor(.75 * geometry.height) self.set_default_size(width, height) else: # Guess a reasonable size self.set_default_size(600, 800) self.set_size_request(100, 100) # Minimum size def set_window_icon(self) -> None: """ Attempts to find a generic RSS icon in the user's GTK theme and associates it with the program if found. """ try: theme = Gtk.IconTheme.get_default() icon = theme.lookup_icon( 'rss', 32, Gtk.IconLookupFlags.GENERIC_FALLBACK) if icon: icon = icon.load_icon() self.set_icon(icon) except GLib.GError: # No RSS icon found pass def switch_view(self, news_store: NewsStore, preferences: Preferences) -> NewsView: """ Activates the view currently chosen in the preferences and returns it. """ appearance_prefs = preferences.appearance_preferences() view_key = appearance_prefs['View'] views = {'Two-Pane': TwoPaneView, 'Three-Pane': ThreePaneView} view_class = TwoPaneView # views[view_key] # FIXME: Hardcoded for development if type(self.current_view) != view_class: # Ensure not switching to same view. if self.current_view: self.current_view.destroy_display() self.current_view = view_class(news_store, appearance_prefs) self.add(self.current_view.top_level()) self.show_all() self.get_preferred_size() # TODO: Investigate if still needed. return self.current_view def create_css(self) -> Gtk.CssProvider: css_provider = Gtk.CssProvider() css_provider.load_from_data(self.preferences.get_appearance_css()) context = Gtk.StyleContext() context.add_provider_for_screen( self.get_screen(), css_provider, Gtk.STYLE_PROVIDER_PRIORITY_APPLICATION) return css_provider def update_css(self) -> None: self.css_provider.load_from_data(self.preferences.get_appearance_css()) def create_header(self) -> Gtk.HeaderBar: header_bar = Gtk.HeaderBar(show_close_button=True) header_bar.pack_start(self.create_add_button()) header_bar.pack_start(self.create_preferences_button()) header_bar.pack_start(self.create_refresh_button()) self.set_titlebar(header_bar) return header_bar def create_add_button(self) -> Gtk.Button: add_button = make_button( theme_icon_string='add', tooltip='Quickly add a feed', signal='clicked', signal_func=self.on_add_clicked) return add_button def create_preferences_button(self) -> Gtk.Button: preferences_button = make_button( theme_icon_string='gtk-preferences', backup_icon_string='preferences-system', tooltip='Preferences', signal='clicked', signal_func=self.on_preferences_clicked) return preferences_button def create_refresh_button(self) -> Gtk.Button: refresh_button = make_button( theme_icon_string='view-refresh', tooltip='Refresh', signal='clicked', signal_func=self.on_refresh_clicked) refresh_button.set_focus_on_click(False) return refresh_button def on_add_clicked(self, widget: Gtk.Widget = None) -> None: dialog = FeedDialog(self, self.preferences.feeds()) feed = dialog.get_response() if feed: self.preferences.add_feed(feed) self.on_refresh_clicked() # Do a convenience refresh def on_preferences_clicked(self, widget: Gtk.Widget = None) -> None: pw = PreferencesWindow(self, self.preferences, self.cache) response = pw.run() if response == Gtk.ResponseType.OK: pw.apply_choices() pw.destroy() def on_refresh_clicked(self, widget: Gtk.Widget = None) -> None: """ Goal: 1. Take each feed URI and look for current items. 2. Only scrape item URIs not in cache. """ ''' self.news_store.clear() self.gatherer.request(None) ''' for feed in self.preferences.feeds().values(): self.gatherer.request(feed) @staticmethod def do_scroll(widget: Gtk.Widget, scroll: Gtk.ScrollType) -> None: try: widget.do_scroll_child(widget, scroll, False) except AttributeError: pass def on_key_press(self, widget: Gtk.Widget, event: Gdk.EventKey) -> None: key = Gdk.keyval_name(event.keyval) if key == 'F5': self.on_refresh_clicked() elif key == 'Left': self.current_view.change_position(-1) elif key == 'Right': self.current_view.change_position(1) elif key == 'Up': self.do_scroll(widget, Gtk.ScrollType.STEP_BACKWARD) elif key == 'Down': self.do_scroll(widget, Gtk.ScrollType.STEP_FORWARD) elif key == 'Return': if event.state & Gdk.ModifierType.CONTROL_MASK: self.current_view.get_then_open_link() else: self.current_view.change_position(0) else: pass
def __init__(self, *args): visualizer.File.__init__(self, *args) self.gatherer = Gatherer() self.x_scope = DynamicScope()
class Waves(visualizer.Visualizer): def __init__(self, args): self._gathered_segments_layer = None visualizer.Visualizer.__init__(self, args, file_class=File, peer_class=Peer, segment_class=Segment) if self.args.peer_info: self.layout_managers = {"left": LayoutManager1d(), "right": LayoutManager1d()} self.waves_margin = self.parse_margin_argument(self.args.waves_margin) @staticmethod def add_parser_arguments(parser): visualizer.Visualizer.add_parser_arguments(parser) parser.add_argument("--peer-info", action="store_true") parser.add_argument("--enable-title", action="store_true") parser.add_argument("--title-size", type=float, default=30.0) parser.add_argument("--test-title", type=str) visualizer.Visualizer.add_margin_argument(parser, "--waves-margin") def resized_window(self): self.waves_margin.update() self._waves_width = self.width - self.waves_margin.left - self.waves_margin.right self._waves_right = self.width - self.waves_margin.right self._waves_height = self.height - self.waves_margin.top - self.waves_margin.bottom self._waves_top = self.height - self.waves_margin.top self._title_renderer = None def configure_2d_projection(self): glMatrixMode(GL_PROJECTION) glLoadIdentity() glOrtho(0.0, self.window_width, 0.0, self.window_height, -1.0, 1.0) glMatrixMode(GL_MODELVIEW) def synth_address_received(self): self.subscribe_to_waveform() def reset(self): visualizer.Visualizer.reset(self) self.playing_segments = collections.OrderedDict() self.gatherer = Gatherer() if self._gathered_segments_layer: self._gathered_segments_layer.refresh() self._first_segment_received = False self._title_renderer = None def pan_segment(self, segment): # let orchestra & synth spatialize pass def InitGL(self): visualizer.Visualizer.InitGL(self) glClearColor(0.0, 0.0, 0.0, 0.0) self._gathered_segments_layer = self.new_layer(self._render_gathered_segments) def update(self): outdated = [] for segment in self.playing_segments.values(): if not segment.is_playing(): self.gatherer.add(segment) outdated.append(segment.id) if len(outdated) > 0: for segment_id in outdated: self.playing_segments[segment_id].free() del self.playing_segments[segment_id] self._gathered_segments_layer.refresh() def render(self): self._set_gathered_color() self._gathered_segments_layer.draw() self.draw_playing_segments() if self.args.enable_title: if not self._title_renderer and (self._first_segment_received or self.args.test_title): self._create_title_renderer() if self._title_renderer: self._render_title() def _create_title_renderer(self): if self.args.test_title: title = self.args.test_title else: title = self.torrent_title size = self.args.title_size / 1024 * self.width self._title_renderer = TitleRenderer(title, size, self) def _render_title(self): if self.download_completed(): color = self._fade_out_color() else: color = Vector3d(1,1,1) glColor3f(*color) x = self.waves_margin.left + 10.0 / 640 * self.width y = self.height * 0.03 self._title_renderer.render(x, y) def _set_gathered_color(self): if self.download_completed(): self.gathered_color = self._fade_out_color() self._gathered_segments_layer.refresh() elif self.torrent_length > 0: torrent_progress = float(self.gatherer.gathered_bytes()) / self.torrent_length self.gathered_color = GATHERED_COLOR + (WAVEFORM_COLOR - GATHERED_COLOR) * pow(torrent_progress, 20) else: self.gathered_color = GATHERED_COLOR def _fade_out_color(self): time_after_completion = max(self.now - self.torrent_download_completion_time, 0) if time_after_completion > FADE_OUT_DURATION: return Vector3d(0,0,0) else: return WAVEFORM_COLOR * pow(1 - time_after_completion/FADE_OUT_DURATION, 0.15) def active(self): return len(self.playing_segments) > 0 def finished(self): if self.download_completed(): time_after_completion = max(self.now - self.torrent_download_completion_time, 0) if time_after_completion > FADE_OUT_DURATION: return True def draw_playing_segments(self): glEnable(GL_LINE_SMOOTH) glHint(GL_LINE_SMOOTH_HINT, GL_NICEST) glEnable(GL_BLEND) glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA) for segment in self.playing_segments.values(): segment.render() def _render_gathered_segments(self): glBegin(GL_QUADS) x1 = self.waves_margin.left x2 = self._waves_right min_height = GATHERED_LINE_WIDTH * self._waves_height for segment in self.gatherer.pieces(): y1 = self.byte_to_py(segment.torrent_begin) y2 = max(self.byte_to_py(segment.torrent_end), y1 + min_height) if (y2 - y1) > min_height: d = min((y2 - y1) * 0.2, MAX_GRADIENT_HEIGHT * self._waves_height) y1d = y1 + d y2d = y2 - d glColor3f(0, 0, 0) glVertex2f(x1, y1) glColor3f(*self.gathered_color) glVertex2f(x1, y1d) glVertex2f(x2, y1d) glColor3f(0, 0, 0) glVertex2f(x2, y1) glColor3f(0, 0, 0) glVertex2f(x1, y2) glColor3f(*self.gathered_color) glVertex2f(x1, y2d) glVertex2f(x2, y2d) glColor3f(0, 0, 0) glVertex2f(x2, y2) glColor3f(*self.gathered_color) glVertex2f(x1, y1d) glVertex2f(x1, y2d) glVertex2f(x2, y2d) glVertex2f(x2, y1d) else: glColor3f(0, 0, 0) glVertex2f(x1, y1) glColor3f(*self.gathered_color) glVertex2f(x1, y2) glVertex2f(x2, y2) glColor3f(0, 0, 0) glVertex2f(x2, y1) glEnd() def byte_to_py(self, byte): return int(self.waves_margin.bottom + self.byte_to_relative_y(byte) * self._waves_height) def byte_to_relative_y(self, byte): return float(byte) / self.torrent_length def handle_segment_waveform_value(self, segment, value): segment.append_to_waveform(value)