class GathererTest(unittest.TestCase):
def test_expected_order(self):
for chunk in self.tr_log.chunks:
self._gather(chunk)
self.assertEquals(1, len(self.gatherer.pieces()))
def test_random_order(self):
while len(self.tr_log.chunks) > 0:
index = random.randint(0, len(self.tr_log.chunks)-1)
chunk = self.tr_log.chunks[index]
del self.tr_log.chunks[index]
self._gather(chunk)
self.assertEquals(1, len(self.gatherer.pieces()))
def setUp(self):
self.tr_log = TrLogReader(SESSION_FILENAME).get_log()
#self.segments = Interpreter().interpret(self.tr_log.chunks, self.tr_log.files)
self.gatherer = Gatherer()
self.files = [File(f["offset"], f["length"]) for f in self.tr_log.files]
def _gather(self, piece_info):
f = self.files[piece_info["filenum"]]
begin = piece_info["begin"] - f.offset
end = piece_info["end"] - f.offset
self.gatherer.add(Piece(begin, end, f))
def test_add_second_piece(self):
gatherer = Gatherer()
gatherer.add(Piece(10, 20))
gatherer.add(Piece(40, 50))
expected_pieces = [Piece(10, 20),
Piece(40, 50)]
self.assertEquals(expected_pieces, gatherer.pieces())
def sysinfo(interval=5, host='localhost', port=35367):
"""Takes display name, port and update interval as arguments."""
g = Gatherer()
d = DisplayClient(host, port)
def backlight_switch(signum, frame):
d.backlight()
date = strftime("Date: %a %b %d, %Y ")
d.write(1,1,string="%s %s %s %s" % (g.hostname, g.system, g.system_release, g.machine))
d.write(1,26,string=date)
d.write(1,15,string="Uptime: ")
d.write(1,37,string="Time: ")
d.write(1,48,string="Load: ")
try:
while True:
signal(SIGUSR1, backlight_switch)
g.refresh()
if date != strftime("Date: %a %b %d, %Y "):
date = strftime("Date: %a %b %d, %Y ")
d.write(1,26,string=date)
d.write(51,15,string="%s " % ' '.join(g.uptime.split('up')[1].split('user')[0].split()[:-1])[:-1])
d.write(35,37,string=strftime("%H:%M"))
d.write(35,48,string=g.uptime.split('average:')[-1].strip())
sleep(interval)
except KeyboardInterrupt:
d.close()
class File:
def __init__(self, length, visualizer):
self.visualizer = visualizer
self.arriving_chunks = OrderedDict()
self.gatherer = Gatherer()
self.radius = 50.0
self.x = random.uniform(self.radius, visualizer.width - self.radius*2)
self.y = random.uniform(self.radius, visualizer.height - self.radius*2)
def add_chunk(self, chunk):
chunk.boid = Boid(self.get_departure_position(chunk), 10.0, 3.0)
chunk.arrival_position = self.get_arrival_position(chunk)
chunk.boid.arrive(chunk.arrival_position)
chunk.arrived = False
self.arriving_chunks[chunk.id] = chunk
def stopped_playing(self, chunk_id):
chunk = self.arriving_chunks[chunk_id]
del self.arriving_chunks[chunk_id]
self.gatherer.add(chunk)
def get_departure_position(self, chunk):
if chunk.pan < 0.5:
x = 0
else:
x = self.visualizer.width
y = chunk.height * self.visualizer.height
return Vector2d(x, y)
def get_arrival_position(self, chunk):
angle = 2 * math.pi * chunk.begin / chunk.file_length
x = self.x + self.radius * math.cos(angle)
y = self.y + self.radius * math.sin(angle)
return Vector2d(x, y)
def process_medialist(self, medialist, alwaysnotify=False, stop_on_error=False):
self.init_run(len(medialist) > 0)
processed = {'tvshow': {}, 'movie': [], 'episode': []}
artcount = 0
currentitem = 0
if medialist:
gatherer = Gatherer(self.monitor, self.only_filesystem)
for mediaitem in medialist:
if not mediainfo.is_known_mediatype(mediaitem):
continue
log('Processing {0}'.format(mediaitem['label']), xbmc.LOGINFO)
if self.visible:
self.progress.update(currentitem * 100 // len(medialist), message=mediaitem['label'])
currentitem += 1
self.add_additional_iteminfo(mediaitem)
cleaned = mediaitem.get_artwork_updates(cleaner.clean_artwork(mediaitem), mediaitem['art'])
if cleaned:
add_art_to_library(mediaitem['mediatype'], mediaitem.get('seasons'), mediaitem['dbid'], cleaned)
mediaitem['art'].update(cleaned)
mediaitem['art'] = dict(item for item in mediaitem['art'].iteritems() if item[1])
forcedart, availableart, services_hit, error = gatherer.getartwork(mediaitem)
if error:
header = L(PROVIDER_ERROR_MESSAGE).format(error['providername'])
xbmcgui.Dialog().notification(header, error['message'], xbmcgui.NOTIFICATION_WARNING)
log('{0}\n{1}'.format(header, error['message']))
if stop_on_error:
break
else:
add_processeditem(processed, mediaitem)
for arttype, imagelist in availableart.iteritems():
self.sort_images(arttype, imagelist, mediaitem['file'])
existingart = dict(mediaitem['art'])
# Remove existing local artwork if it is no longer available
localart = [(arttype, image['url']) for arttype, image in forcedart.iteritems() \
if not image['url'].startswith('http')]
selectedart = dict((arttype, None) for arttype, url in existingart.iteritems() \
if not url.startswith('http') and (arttype, url) not in localart)
selectedart.update((key, image['url']) for key, image in forcedart.iteritems())
selectedart = mediainfo.renumber_all_artwork(selectedart)
existingart.update(selectedart)
selectedart.update(self.get_top_missing_art(mediaitem['mediatype'], existingart, availableart, mediaitem.get('seasons')))
selectedart = mediainfo.get_artwork_updates(mediaitem['art'], selectedart)
if selectedart:
add_art_to_library(mediaitem['mediatype'], mediaitem.get('seasons'), mediaitem['dbid'], selectedart)
artcount += len(selectedart)
if not services_hit:
if self.monitor.abortRequested():
break
elif self.monitor.waitForAbort(THROTTLE_TIME):
break
self.finish_run()
if alwaysnotify or artcount:
notifycount(artcount)
return processed
def __init__(self, length, visualizer):
self.visualizer = visualizer
self.arriving_chunks = OrderedDict()
self.gatherer = Gatherer()
self.radius = 50.0
self.x = random.uniform(self.radius, visualizer.width - self.radius*2)
self.y = random.uniform(self.radius, visualizer.height - self.radius*2)
def __init__(self, *args):
visualizer.File.__init__(self, *args)
self.position = self.visualizer.position_for_new_file()
self.gatherer = Gatherer()
self.inner_radius = self.visualizer.scale(RADIUS - CIRCLE_THICKNESS)
self.radius = self.visualizer.scale(RADIUS)
self.velocity = Vector2d(0,0)
def scrape():
scraper = LinkedinScraper()
scraper.login(os.environ['LINKEDIN_USER'], os.environ['LINKEDIN_PW'])
gatherer = Gatherer(scraper)
watchdog = Watchdog(gatherer)
job_list = [
'data intern', 'machine learning intern', 'software intern',
'python intern'
]
location_list = [
'Vancouver',
'New York',
'San Francisco Bay Area',
'Toronto',
]
for job in job_list:
for location in location_list:
print('gathering for', job, location)
watchdog.monitor_gather(job, location)
for job in job_list:
print('gathering for', job, 'Worldwide')
watchdog.monitor_gather(job, 'Worldwide')
return 'Done'
def __init__(self, args):
visualizer.Visualizer.__init__(self, args,
file_class=File,
peer_class=Peer,
segment_class=Segment)
self.inner_x = WALL_X
self.outer_x = WALL_X + STAIRS_WIDTH
self.wall_rear_x = WALL_X - WALL_WIDTH
self.wall_bottom = self.step_y(NUM_STEPS) - STEP_HEIGHT
self.stairs_depth = self.step_z(NUM_STEPS)
self.files = {}
self.segments = {}
self.gatherer = Gatherer()
self._segments_split_at_step_boundaries = None
self._dragging_orientation = False
self._dragging_y_position = False
self._set_camera_position(CAMERA_POSITION)
self._set_camera_orientation(CAMERA_Y_ORIENTATION, CAMERA_X_ORIENTATION)
self.enable_accum()
self.enable_3d()
if self.args.waveform:
self.gathered_color_v = CURSOR_COLOR_V * GATHERED_OPACITY + STEPS_COLOR_V * (1 - GATHERED_OPACITY)
self.gathered_color_h = CURSOR_COLOR_H * GATHERED_OPACITY + STEPS_COLOR_H * (1 - GATHERED_OPACITY)
self.subscribe_to_waveform()
else:
self.gathered_color_v = GATHERED_COLOR_V
self.gathered_color_h = GATHERED_COLOR_H
self.subscribe_to_amp()
def __init__(self, preferences: Preferences, cache: Cache, **kwargs):
Gtk.Window.__init__(self, **kwargs)
self.preferences = preferences
self.cache = cache
self.news_store = NewsStore()
# self.gatherer = StubGatherer(self.news_store)
self.gatherer = Gatherer(self.news_store)
self.connect_signals()
self.prepare_appearance()
self.css_provider = self.create_css()
self.current_view = None
self.switch_view(self.news_store, self.preferences)
def __init__(self, args):
visualizer.Visualizer.__init__(self, args,
file_class=File,
peer_class=Peer,
segment_class=Segment)
self.subscribe_to_waveform()
self.playing_segments = collections.OrderedDict()
self.gatherer = Gatherer()
def reset(self):
visualizer.Visualizer.reset(self)
self.playing_segments = collections.OrderedDict()
self.gatherer = Gatherer()
if self._gathered_segments_layer:
self._gathered_segments_layer.refresh()
self._first_segment_received = False
self._title_renderer = None
def __init__(self, *args):
visualizer.File.__init__(self, *args)
self.gatherer = Gatherer()
self.y = self.visualizer.step_y(self.filenum+1)
self.z1 = self.visualizer.step_z(self.filenum)
self.z2 = self.visualizer.step_z(self.filenum+1)
self.z = (self.z1 + self.z2) / 2
self.neighbour_y = self.visualizer.step_y(self.filenum+2)
self.neighbour_z1 = self.visualizer.step_z(self.filenum+1)
class File(visualizer.File):
def __init__(self, *args):
visualizer.File.__init__(self, *args)
self.gatherer = Gatherer()
self.x_scope = DynamicScope()
def add_segment(self, segment):
self.x_scope.put(segment.begin)
self.x_scope.put(segment.end)
segment.pan = (self.x_scope.map(segment.begin) + self.x_scope.map(segment.end)) / 2
segment.departure_position = segment.peer_position()
self.visualizer.playing_segment(segment, segment.pan)
def render(self):
self.x_scope.update()
self.draw_gathered_segments()
def draw_gathered_segments(self):
for segment in self.gatherer.pieces():
segment.draw_gathered()
def byte_to_step_position(self, byte):
return float(byte) / self.length * NUM_STEPS
def byte_to_wall_position(self, byte):
step_position = self.byte_to_step_position(byte)
step = int(step_position)+1
return Vector2d(self.visualizer.step_z(step_position),
self.visualizer.step_y(step))
def byte_surfaces(self, begin, end, min_size=None):
step_pos1 = self.byte_to_step_position(begin)
step_pos2 = self.byte_to_step_position(end)
if (step_pos2 - step_pos1) < min_size:
step_pos1, step_pos2 = self.resize(step_pos1, step_pos2, min_size)
step1 = int(step_pos1)
step2 = int(step_pos2)
fraction1 = step_pos1 % 1
fraction2 = step_pos2 % 1
for step in range(step1, step2+1):
if step == step1:
relative_begin = fraction1
else:
relative_begin = 0
if step == step2:
relative_end = fraction2
else:
relative_end = 1
step_surfaces = list(self.visualizer.step_surfaces(step, relative_begin, relative_end))
yield step_surfaces[1]
def resize(self, pos1, pos2, new_size):
new_pos1 = max(pos1 - new_size/2, 0)
new_pos2 = min(pos2 + new_size/2, NUM_STEPS)
return new_pos1, new_pos2
def test_overlap_multiple_pieces(self):
gatherer = Gatherer()
gatherer.add(Piece(10, 20))
gatherer.add(Piece(30, 40))
gatherer.add(Piece(0, 50))
expected_pieces = [Piece(0, 50)]
self.assertEquals(expected_pieces, gatherer.pieces())
def test_fit_hole(self):
gatherer = Gatherer()
gatherer.add(Piece(10, 20))
gatherer.add(Piece(40, 50))
gatherer.add(Piece(20, 40))
expected_pieces = [Piece(10, 50)]
self.assertEquals(expected_pieces, gatherer.pieces())
def test_prepend(self):
gatherer = Gatherer()
gatherer.add(Piece(10, 20))
gatherer.add(Piece(40, 50))
gatherer.add(Piece(30, 40))
expected_pieces = [Piece(10, 20),
Piece(30, 50)]
self.assertEquals(expected_pieces, gatherer.pieces())
def test_multiple_files_append(self):
f1 = File(offset=0, length=30)
f2 = File(offset=30, length=20)
gatherer = Gatherer()
gatherer.add(Piece(20, 30, f1))
gatherer.add(Piece(0, 10, f2))
expected_pieces = [Piece(20, 40, f1)]
self.assertEquals(expected_pieces, gatherer.pieces())
class File:
def __init__(self, length, visualizer):
self.visualizer = visualizer
self.arriving_chunks = OrderedDict()
self.gatherer = Gatherer()
def add_chunk(self, chunk):
self.arriving_chunks[chunk.id] = chunk
chunk.setup()
def update(self):
self.update_arriving_chunks()
def update_arriving_chunks(self):
for chunk in self.arriving_chunks.values():
chunk.update()
if chunk.arrived():
self.gather_chunk(chunk)
def gather_chunk(self, chunk):
del self.arriving_chunks[chunk.id]
chunk.has_arrived = True
self.gatherer.add(chunk)
def sysinfo(interval=5, host='localhost', port=35367):
"""Takes display name, port and update interval as arguments."""
g = Gatherer()
d = DisplayClient(host, port)
def backlight_switch(signum, frame):
d.backlight()
date = strftime("Date: %a %b %d, %Y ")
d.write(1,
1,
string="%s %s %s %s" %
(g.hostname, g.system, g.system_release, g.machine))
d.write(1, 26, string=date)
d.write(1, 15, string="Uptime: ")
d.write(1, 37, string="Time: ")
d.write(1, 48, string="Load: ")
try:
while True:
signal(SIGUSR1, backlight_switch)
g.refresh()
if date != strftime("Date: %a %b %d, %Y "):
date = strftime("Date: %a %b %d, %Y ")
d.write(1, 26, string=date)
d.write(
51,
15,
string="%s " % ' '.join(
g.uptime.split('up')[1].split('user')[0].split()[:-1])[:-1]
)
d.write(35, 37, string=strftime("%H:%M"))
d.write(35, 48, string=g.uptime.split('average:')[-1].strip())
sleep(interval)
except KeyboardInterrupt:
d.close()
def __init__(self, preferences, cache, **kwargs):
Gtk.Window.__init__(self, **kwargs)
# Signals
self.connect('key_press_event', self.on_key_press)
self.connect('item_scraped_event', self.on_item_scraped)
self.connect('feed_gathered_event', self.on_feed_gathered)
self.set_good_default_size()
self.set_window_icon()
self.preferences = preferences
self.css_provider = self.create_css()
self.header_bar = self.create_header()
self.cache = cache
self.gatherer = Gatherer(self, self.preferences, self.cache)
self.current_view = None
self.switch_view()
self.gatherer.request_feeds()
class File(visualizer.File):
def __init__(self, *args):
visualizer.File.__init__(self, *args)
self.gatherer = Gatherer()
self.x_scope = DynamicScope()
def add_segment(self, segment):
self.x_scope.put(segment.begin)
self.x_scope.put(segment.end)
segment.pan = (self.x_scope.map(segment.begin) + self.x_scope.map(segment.end)) / 2
segment.departure_position = segment.peer_position()
self.visualizer.playing_segment(segment)
def render(self):
self.x_scope.update()
self.draw_gathered_segments()
def draw_gathered_segments(self):
for segment in self.gatherer.pieces():
segment.draw_gathered()
def byte_to_coord(self, byte):
return self.visualizer.prepend_margin_width + \
self.x_scope.map(byte) * self.visualizer.safe_width
class File(visualizer.File):
def __init__(self, *args):
visualizer.File.__init__(self, *args)
self.gatherer = Gatherer()
self.y = self.visualizer.step_y(self.filenum+1)
self.z1 = self.visualizer.step_z(self.filenum)
self.z2 = self.visualizer.step_z(self.filenum+1)
self.z = (self.z1 + self.z2) / 2
self.neighbour_y = self.visualizer.step_y(self.filenum+2)
self.neighbour_z1 = self.visualizer.step_z(self.filenum+1)
def add_segment(self, segment):
segment.departure_position = segment.peer.position
self.visualizer.playing_segment(segment)
def render(self):
self.draw_gathered_segments()
def draw_gathered_segments(self):
for segment in self.gatherer.pieces():
segment.draw_gathered()
def byte_to_x(self, byte):
return WALL_X + float(byte) / self.length * STAIRS_WIDTH
def reset(self):
visualizer.Visualizer.reset(self)
self.playing_segments = collections.OrderedDict()
self.gatherer = Gatherer()
def __init__(self, *args):
visualizer.File.__init__(self, *args)
self.playing_segments = OrderedDict()
self.gatherer = Gatherer()
self.x_scope = DynamicScope()
class Waves(visualizer.Visualizer):
def __init__(self, args):
visualizer.Visualizer.__init__(self, args,
file_class=File,
segment_class=Segment)
self.subscribe_to_waveform()
def reset(self):
visualizer.Visualizer.reset(self)
self.playing_segments = collections.OrderedDict()
self.gatherer = Gatherer()
def pan_segment(self, segment):
# let orchestra & synth spatialize
pass
def InitGL(self):
visualizer.Visualizer.InitGL(self)
glClearColor(0.0, 0.0, 0.0, 0.0)
self._gathered_segments_layer = self.new_layer(self._render_gathered_segments)
def update(self):
outdated = []
for segment in self.playing_segments.values():
if not segment.is_playing():
self.gatherer.add(segment)
outdated.append(segment.id)
if len(outdated) > 0:
for segment_id in outdated:
del self.playing_segments[segment_id]
self._gathered_segments_layer.refresh()
def render(self):
self.update()
self._gathered_segments_layer.draw()
self.draw_playing_segments()
def draw_playing_segments(self):
glEnable(GL_LINE_SMOOTH)
glHint(GL_LINE_SMOOTH_HINT, GL_NICEST)
glEnable(GL_BLEND)
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
for segment in self.playing_segments.values():
segment.render()
def _render_gathered_segments(self):
glBegin(GL_QUADS)
x1 = 0
x2 = self.width
min_height = GATHERED_LINE_WIDTH * self.height
for segment in self.gatherer.pieces():
y1 = self.byte_to_py(segment.torrent_begin)
y2 = max(self.byte_to_py(segment.torrent_end), y1 + min_height)
if (y2 - y1) > min_height:
d = min((y2 - y1) * 0.2, MAX_GRADIENT_HEIGHT * self.height)
y1d = y1 + d
y2d = y2 - d
glColor3f(0, 0, 0)
glVertex2f(x1, y1)
glColor3f(*GATHERED_COLOR)
glVertex2f(x1, y1d)
glVertex2f(x2, y1d)
glColor3f(0, 0, 0)
glVertex2f(x2, y1)
glColor3f(0, 0, 0)
glVertex2f(x1, y2)
glColor3f(*GATHERED_COLOR)
glVertex2f(x1, y2d)
glVertex2f(x2, y2d)
glColor3f(0, 0, 0)
glVertex2f(x2, y2)
glColor3f(*GATHERED_COLOR)
glVertex2f(x1, y1d)
glVertex2f(x1, y2d)
glVertex2f(x2, y2d)
glVertex2f(x2, y1d)
else:
glColor3f(0, 0, 0)
glVertex2f(x1, y1)
glColor3f(*GATHERED_COLOR)
glVertex2f(x1, y2)
glVertex2f(x2, y2)
glColor3f(0, 0, 0)
glVertex2f(x2, y1)
glEnd()
def byte_to_py(self, byte):
return int(self.byte_to_relative_y(byte) * self.height)
def byte_to_relative_y(self, byte):
return float(byte) / self.torrent_length
def handle_segment_waveform_value(self, segment, value):
segment.waveform.appendleft(value)
def finished(self):
if len(self.gatherer.pieces()) == 1:
piece = self.gatherer.pieces()[0]
return piece.begin == 0 and piece.end == self.torrent_length
else:
return False
class File:
def __init__(self, length, visualizer):
self.visualizer = visualizer
self.arriving_chunks = OrderedDict()
self.gatherer = Gatherer()
def add_chunk(self, chunk):
chunk.boid = Boid(self.get_departure_position(chunk), 10.0, 3.0)
chunk.target = None
chunk.target_type = None
chunk.target_position = None
chunk.target_position = self.get_target_position(chunk)
chunk.boid.arrive(chunk.target_position)
chunk.arrived = False
self.arriving_chunks[chunk.id] = chunk
def stopped_playing(self, chunk_id):
chunk = self.arriving_chunks[chunk_id]
self.gather_chunk(chunk)
def gather_chunk(self, chunk):
del self.arriving_chunks[chunk.id]
# TODO: verify that these positions are really OK
chunk.begin_position = Vector(chunk.boid.loc.x, chunk.boid.loc.y)
chunk.end_position = Vector(chunk.boid.loc.x, chunk.boid.loc.y)
self.gatherer.add(chunk)
self.reorient_chunks_following(chunk)
def get_departure_position(self, chunk):
if chunk.pan < 0.5:
x = 0
else:
x = self.visualizer.width
y = chunk.height * self.visualizer.height
return Vector(x, y)
def get_target_position(self, chunk):
if not chunk.target_position:
self.find_target(chunk)
return chunk.target_position
def find_target(self, chunk):
position = self.find_joinable_piece(chunk)
if position:
chunk.target_type = TargetType.PIECE
chunk.target_position = position
return
other_chunk = self.find_other_huntable_chunk(chunk)
if other_chunk:
chunk.target_type = TargetType.CHUNK
chunk.target = other_chunk
chunk.target_position = other_chunk.boid.loc
return
if len(self.gatherer.pieces()) > 0:
chunk.target_position = self.close_to_existing_piece()
return
chunk.target_type = TargetType.NEW_PIECE
chunk.target_position = self.anywhere()
def anywhere(self):
return Vector(random.uniform(self.visualizer.width * INNER_MARGIN,
self.visualizer.width * (1 - INNER_MARGIN*2)),
random.uniform(self.visualizer.height * INNER_MARGIN,
self.visualizer.height * (1 - INNER_MARGIN*2)))
def close_to_existing_piece(self):
piece = random.choice(self.gatherer.pieces())
position = random.choice([piece.begin_position,
piece.end_position])
angle = random.uniform(0, 2 * math.pi)
distance = 1.0
movement = Vector(distance * math.cos(angle),
distance * math.sin(angle))
return position + movement
def find_joinable_piece(self, chunk):
appendable_piece_key = self.gatherer.find_appendable_piece(chunk)
prependable_piece_key = self.gatherer.find_prependable_piece(chunk)
if appendable_piece_key and prependable_piece_key:
appendable_position = self.gatherer.piece(appendable_piece_key).begin_position
prepandable_position = self.gatherer.piece(prependable_piece_key).end_position
chunk.target_position = (appendable_position + prepandable_position) / 2
elif appendable_piece_key:
return self.gatherer.piece(appendable_piece_key).begin_position
elif prependable_piece_key:
return self.gatherer.piece(prependable_piece_key).end_position
def find_other_huntable_chunk(self, chunk):
# optimization: iterate appendables in reverse order, or use some kind of cache
for other in self.arriving_chunks.values():
if other.end == chunk.begin:
return other
if other.begin == chunk.end:
return other
def reorient_chunks_following(self, targeted_chunk):
for chunk in self.arriving_chunks.values():
if chunk.target_type == TargetType.CHUNK and \
chunk.target == targeted_chunk:
self.find_target(chunk)
def update(self):
self.update_pieces()
self.update_arriving_chunks()
def update_pieces(self):
for piece in self.gatherer.pieces():
self.get_piece_force(piece)
for piece in self.gatherer.pieces():
self.apply_piece_force(piece)
def get_piece_force(self, piece):
piece.desired_length = self.desired_piece_length(piece)
piece.begin_force = Vector(0,0)
piece.end_force = Vector(0,0)
self.consider_desired_length(piece, piece.begin_force, piece.begin_position, piece.end_position)
self.consider_desired_length(piece, piece.end_force, piece.end_position, piece.begin_position)
piece.begin_force.limit(3.0)
piece.end_force.limit(3.0)
def desired_piece_length(self, piece):
return piece.byte_size / 10
def consider_desired_length(self, piece, force, position, opposite_position):
force += spring_force(position, opposite_position, piece.desired_length)
def apply_piece_force(self, piece):
piece.begin_position += piece.begin_force
piece.end_position += piece.end_force
def update_arriving_chunks(self):
for chunk in self.arriving_chunks.values():
if not chunk.arrived:
chunk.boid.update()
if self.arrived(chunk):
#self.visualizer.play_chunk(chunk) # TEMP
chunk.arrived = True
self.gather_chunk(chunk) # TEMP
def arrived(self, chunk):
if chunk.target_type in [TargetType.PIECE,
TargetType.NEW_PIECE]:
distance = (chunk.target_position - chunk.boid.loc).mag()
return distance < 1.0
else:
return False
class File(visualizer.File):
def __init__(self, *args):
visualizer.File.__init__(self, *args)
self.playing_segments = OrderedDict()
self.gatherer = Gatherer()
self.x_scope = DynamicScope()
def add_segment(self, segment):
self.x_scope.put(segment.begin)
self.x_scope.put(segment.end)
pan = (self.byte_to_coord(segment.begin) + self.byte_to_coord(segment.end)) / 2
self.visualizer.playing_segment(segment, pan)
self.playing_segments[segment.id] = segment
def update(self):
outdated = filter(lambda segment_id: self.playing_segments[segment_id].relative_age() > 1,
self.playing_segments)
for segment_id in outdated:
self.gatherer.add(self.playing_segments[segment_id])
del self.playing_segments[segment_id]
self.x_scope.update()
def render(self):
self.y = self.visualizer.filenum_to_y_coord(self.filenum)
self.draw_gathered_segments()
self.draw_playing_segments()
def draw_gathered_segments(self):
for segment in self.gatherer.pieces():
self.draw_segment(segment, 0)
def draw_playing_segments(self):
for segment in self.playing_segments.values():
self.draw_playing_segment(segment)
def draw_playing_segment(self, segment):
actuality = 1 - segment.relative_age()
self.draw_segment(segment, actuality)
def draw_segment(self, segment, actuality):
y_offset = actuality * 10
height = 3 + actuality * 10
y1 = int(self.y + y_offset)
y2 = int(self.y + y_offset + height)
x1 = self.visualizer.prepend_margin_width + \
int(self.byte_to_coord(segment.begin) * self.visualizer.safe_width)
x2 = self.visualizer.prepend_margin_width + \
int(self.byte_to_coord(segment.end) * self.visualizer.safe_width)
x1, x2 = self.upscale(x1, x2, actuality)
if x2 == x1:
x2 = x1 + 1
opacity = 0.2 + (actuality * 0.8)
glColor3f(1-opacity, 1-opacity, 1-opacity)
glBegin(GL_LINE_LOOP)
glVertex2i(x1, y2)
glVertex2i(x2, y2)
glVertex2i(x2, y1)
glVertex2i(x1, y1)
glEnd()
def upscale(self, x1, x2, actuality):
unscaled_size = x2 - x1
desired_size = actuality * ARRIVAL_SIZE
if desired_size > unscaled_size:
mid = (x1 + x2) / 2
half_desired_size = int(desired_size/2)
x1 = mid - half_desired_size
x2 = mid + half_desired_size
return (x1, x2)
def byte_to_coord(self, byte):
return self.x_scope.map(byte)
def test_overlap_begin(self):
gatherer = Gatherer()
gatherer.add(Piece(10, 20))
gatherer.add(Piece(0, 15))
expected_pieces = [Piece(0, 20)]
self.assertEquals(expected_pieces, gatherer.pieces())
from pycallgraph import PyCallGraph
from pycallgraph.output import GraphvizOutput
from gatherer import Gatherer
tickrate = 60
with PyCallGraph(output=GraphvizOutput()):
aaron = Gatherer(startingpos=[380, 280])
aaron.update()
def __init__(self, length, visualizer):
self.visualizer = visualizer
self.arriving_chunks = OrderedDict()
self.gatherer = Gatherer()
def monitor_gather(self, keywords, location, start=0):
while True:
#try:
self.gatherer.gather_jobs(keywords,
location,
start,
progress_file='progress')
break
#except Exception as e:
# print('ERROR!')
# print(e)
# print('********************************')
# start = int(open('progress').read())
if __name__ == "__main__":
import os
from gatherer import Gatherer
from linkedinScraper import LinkedinScraper
scraper = LinkedinScraper()
#scraper.login(os.environ['LINKEDIN_USER'], os.environ['LINKEDIN_PW'])
gatherer = Gatherer(scraper)
watchdog = Watchdog(gatherer)
keywords = 'software intern'
location = 'Worldwide'
watchdog.monitor_gather(keywords, location)
from linkedinScraper import LinkedinScraper
from gatherer import Gatherer
scraper = LinkedinScraper()
scraper.login(input('Username?'), input('Password?'))
gatherer = Gatherer(scraper)
job_list = [
'data intern',
'machine learning intern',
'software intern',
'python intern'
]
location_list = [
'Vancouver',
'New York',
'San Francisco Bay Area',
'Toronto'
]
search_list = []
for job in job_list:
for location in location_list:
print('gathering for', job, location)
gatherer.gather_jobs(job, location)
gatherer.digest_data()
def process_item(self, mediatype, dbid, mode):
if self.processor_busy:
return
if mode == MODE_GUI:
xbmc.executebuiltin('ActivateWindow(busydialog)')
if mediatype == mediatypes.TVSHOW:
mediaitem = quickjson.get_tvshow_details(dbid, tvshow_properties)
elif mediatype == mediatypes.MOVIE:
mediaitem = quickjson.get_movie_details(dbid, movie_properties)
elif mediatype == mediatypes.EPISODE:
mediaitem = quickjson.get_episode_details(dbid, episode_properties)
else:
xbmc.executebuiltin('Dialog.Close(busydialog)')
xbmcgui.Dialog().notification("Artwork Beef", L(NOT_SUPPORTED_MESSAGE).format(mediatype), '-', 6500)
return
if mode == MODE_GUI:
self.init_run()
self.add_additional_iteminfo(mediaitem)
gatherer = Gatherer(self.monitor, self.only_filesystem)
forcedart, availableart, _, error = gatherer.getartwork(mediaitem, False)
if error:
header = L(PROVIDER_ERROR_MESSAGE).format(error['providername'])
xbmcgui.Dialog().notification(header, error['message'], xbmcgui.NOTIFICATION_WARNING)
log('{0}\n{1}'.format(header, error['message']))
for arttype, imagelist in availableart.iteritems():
self.sort_images(arttype, imagelist, mediaitem['file'])
xbmc.executebuiltin('Dialog.Close(busydialog)')
if availableart:
if 'seasons' in mediaitem and 'fanart' in availableart:
for season in mediaitem['seasons'].keys():
unseasoned_backdrops = [dict(art) for art in availableart['fanart'] if not art.get('hasseason')]
key = 'season.{0}.fanart'.format(season)
if key in availableart:
availableart[key].extend(unseasoned_backdrops)
else:
availableart[key] = unseasoned_backdrops
tag_forcedandexisting_art(availableart, forcedart, mediaitem['art'])
selectedarttype, selectedart = prompt_for_artwork(mediaitem['mediatype'],
mediaitem['label'], availableart, self.monitor)
if selectedarttype:
if selectedarttype.startswith('season.'):
gentype = selectedarttype.rsplit('.', 1)[1]
multiselect = mediatypes.artinfo[mediatypes.SEASON].get(gentype, {}).get('multiselect', False)
else:
multiselect = mediatypes.artinfo[mediaitem['mediatype']][selectedarttype]['multiselect']
if multiselect:
existingurls = [url for exacttype, url in mediaitem['art'].iteritems() \
if mediainfo.arttype_matches_base(exacttype, selectedarttype)]
urls_toset = [url for url in existingurls if url not in selectedart[1]]
urls_toset.extend([url for url in selectedart[0] if url not in urls_toset])
selectedart = dict(mediainfo.iter_renumbered_artlist(selectedart, selectedarttype, mediaitem['art'].keys()))
else:
selectedart = {selectedarttype: selectedart}
selectedart = mediainfo.get_artwork_updates(mediaitem['art'], selectedart)
if selectedart:
add_art_to_library(mediaitem['mediatype'], mediaitem.get('seasons'), mediaitem['dbid'], selectedart)
notifycount(len(selectedart))
else:
xbmcgui.Dialog().notification(L(NOT_AVAILABLE_MESSAGE),
L(SOMETHING_MISSING) + ' ' + L(FINAL_MESSAGE), '-', 8000)
self.finish_run()
else:
medialist = [mediaitem]
autoaddepisodes = addon.get_setting('autoaddepisodes_list') if addon.get_setting('episode.fanart') else ()
if mediatype == mediatypes.TVSHOW and mediaitem['imdbnumber'] in autoaddepisodes:
medialist.extend(quickjson.get_episodes(dbid, properties=episode_properties))
self.process_medialist(medialist, True)
class MainWindow(Gtk.Window):
__gtype_name__ = 'TroughWindow'
def __init__(self, preferences: Preferences, cache: Cache, **kwargs):
Gtk.Window.__init__(self, **kwargs)
self.preferences = preferences
self.cache = cache
self.news_store = NewsStore()
# self.gatherer = StubGatherer(self.news_store)
self.gatherer = Gatherer(self.news_store)
self.connect_signals()
self.prepare_appearance()
self.css_provider = self.create_css()
self.current_view = None
self.switch_view(self.news_store, self.preferences)
def connect_signals(self) -> None:
self.connect('key_press_event', self.on_key_press)
def prepare_appearance(self) -> None:
self.set_good_default_size()
self.set_window_icon()
self.create_header()
def set_good_default_size(self) -> None:
screen = self.get_screen()
active_window = screen.get_active_window()
if active_window:
monitor = screen.get_monitor_at_window(active_window)
geometry = screen.get_monitor_geometry(monitor)
width = floor(.60 * geometry.width)
height = floor(.75 * geometry.height)
self.set_default_size(width, height)
else:
# Guess a reasonable size
self.set_default_size(600, 800)
self.set_size_request(100, 100) # Minimum size
def set_window_icon(self) -> None:
"""
Attempts to find a generic RSS icon in the user's GTK theme
and associates it with the program if found.
"""
try:
theme = Gtk.IconTheme.get_default()
icon = theme.lookup_icon(
'rss',
32,
Gtk.IconLookupFlags.GENERIC_FALLBACK)
if icon:
icon = icon.load_icon()
self.set_icon(icon)
except GLib.GError: # No RSS icon found
pass
def switch_view(self, news_store: NewsStore, preferences: Preferences) -> NewsView:
"""
Activates the view currently chosen in the preferences and returns it.
"""
appearance_prefs = preferences.appearance_preferences()
view_key = appearance_prefs['View']
views = {'Two-Pane': TwoPaneView, 'Three-Pane': ThreePaneView}
view_class = TwoPaneView # views[view_key] # FIXME: Hardcoded for development
if type(self.current_view) != view_class: # Ensure not switching to same view.
if self.current_view:
self.current_view.destroy_display()
self.current_view = view_class(news_store, appearance_prefs)
self.add(self.current_view.top_level())
self.show_all()
self.get_preferred_size() # TODO: Investigate if still needed.
return self.current_view
def create_css(self) -> Gtk.CssProvider:
css_provider = Gtk.CssProvider()
css_provider.load_from_data(self.preferences.get_appearance_css())
context = Gtk.StyleContext()
context.add_provider_for_screen(
self.get_screen(),
css_provider,
Gtk.STYLE_PROVIDER_PRIORITY_APPLICATION)
return css_provider
def update_css(self) -> None:
self.css_provider.load_from_data(self.preferences.get_appearance_css())
def create_header(self) -> Gtk.HeaderBar:
header_bar = Gtk.HeaderBar(show_close_button=True)
header_bar.pack_start(self.create_add_button())
header_bar.pack_start(self.create_preferences_button())
header_bar.pack_start(self.create_refresh_button())
self.set_titlebar(header_bar)
return header_bar
def create_add_button(self) -> Gtk.Button:
add_button = make_button(
theme_icon_string='add',
tooltip='Quickly add a feed',
signal='clicked',
signal_func=self.on_add_clicked)
return add_button
def create_preferences_button(self) -> Gtk.Button:
preferences_button = make_button(
theme_icon_string='gtk-preferences',
backup_icon_string='preferences-system',
tooltip='Preferences',
signal='clicked',
signal_func=self.on_preferences_clicked)
return preferences_button
def create_refresh_button(self) -> Gtk.Button:
refresh_button = make_button(
theme_icon_string='view-refresh',
tooltip='Refresh',
signal='clicked',
signal_func=self.on_refresh_clicked)
refresh_button.set_focus_on_click(False)
return refresh_button
def on_add_clicked(self, widget: Gtk.Widget = None) -> None:
dialog = FeedDialog(self, self.preferences.feeds())
feed = dialog.get_response()
if feed:
self.preferences.add_feed(feed)
self.on_refresh_clicked() # Do a convenience refresh
def on_preferences_clicked(self, widget: Gtk.Widget = None) -> None:
pw = PreferencesWindow(self, self.preferences, self.cache)
response = pw.run()
if response == Gtk.ResponseType.OK:
pw.apply_choices()
pw.destroy()
def on_refresh_clicked(self, widget: Gtk.Widget = None) -> None:
"""
Goal:
1. Take each feed URI and look for current items.
2. Only scrape item URIs not in cache.
"""
'''
self.news_store.clear()
self.gatherer.request(None)
'''
for feed in self.preferences.feeds().values():
self.gatherer.request(feed)
@staticmethod
def do_scroll(widget: Gtk.Widget, scroll: Gtk.ScrollType) -> None:
try:
widget.do_scroll_child(widget, scroll, False)
except AttributeError:
pass
def on_key_press(self, widget: Gtk.Widget, event: Gdk.EventKey) -> None:
key = Gdk.keyval_name(event.keyval)
if key == 'F5':
self.on_refresh_clicked()
elif key == 'Left':
self.current_view.change_position(-1)
elif key == 'Right':
self.current_view.change_position(1)
elif key == 'Up':
self.do_scroll(widget, Gtk.ScrollType.STEP_BACKWARD)
elif key == 'Down':
self.do_scroll(widget, Gtk.ScrollType.STEP_FORWARD)
elif key == 'Return':
if event.state & Gdk.ModifierType.CONTROL_MASK:
self.current_view.get_then_open_link()
else:
self.current_view.change_position(0)
else:
pass
def __init__(self, *args):
visualizer.File.__init__(self, *args)
self.gatherer = Gatherer()
self.x_scope = DynamicScope()
class Waves(visualizer.Visualizer):
def __init__(self, args):
self._gathered_segments_layer = None
visualizer.Visualizer.__init__(self, args,
file_class=File,
peer_class=Peer,
segment_class=Segment)
if self.args.peer_info:
self.layout_managers = {"left": LayoutManager1d(),
"right": LayoutManager1d()}
self.waves_margin = self.parse_margin_argument(self.args.waves_margin)
@staticmethod
def add_parser_arguments(parser):
visualizer.Visualizer.add_parser_arguments(parser)
parser.add_argument("--peer-info", action="store_true")
parser.add_argument("--enable-title", action="store_true")
parser.add_argument("--title-size", type=float, default=30.0)
parser.add_argument("--test-title", type=str)
visualizer.Visualizer.add_margin_argument(parser, "--waves-margin")
def resized_window(self):
self.waves_margin.update()
self._waves_width = self.width - self.waves_margin.left - self.waves_margin.right
self._waves_right = self.width - self.waves_margin.right
self._waves_height = self.height - self.waves_margin.top - self.waves_margin.bottom
self._waves_top = self.height - self.waves_margin.top
self._title_renderer = None
def configure_2d_projection(self):
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
glOrtho(0.0, self.window_width, 0.0, self.window_height, -1.0, 1.0)
glMatrixMode(GL_MODELVIEW)
def synth_address_received(self):
self.subscribe_to_waveform()
def reset(self):
visualizer.Visualizer.reset(self)
self.playing_segments = collections.OrderedDict()
self.gatherer = Gatherer()
if self._gathered_segments_layer:
self._gathered_segments_layer.refresh()
self._first_segment_received = False
self._title_renderer = None
def pan_segment(self, segment):
# let orchestra & synth spatialize
pass
def InitGL(self):
visualizer.Visualizer.InitGL(self)
glClearColor(0.0, 0.0, 0.0, 0.0)
self._gathered_segments_layer = self.new_layer(self._render_gathered_segments)
def update(self):
outdated = []
for segment in self.playing_segments.values():
if not segment.is_playing():
self.gatherer.add(segment)
outdated.append(segment.id)
if len(outdated) > 0:
for segment_id in outdated:
self.playing_segments[segment_id].free()
del self.playing_segments[segment_id]
self._gathered_segments_layer.refresh()
def render(self):
self._set_gathered_color()
self._gathered_segments_layer.draw()
self.draw_playing_segments()
if self.args.enable_title:
if not self._title_renderer and (self._first_segment_received or self.args.test_title):
self._create_title_renderer()
if self._title_renderer:
self._render_title()
def _create_title_renderer(self):
if self.args.test_title:
title = self.args.test_title
else:
title = self.torrent_title
size = self.args.title_size / 1024 * self.width
self._title_renderer = TitleRenderer(title, size, self)
def _render_title(self):
if self.download_completed():
color = self._fade_out_color()
else:
color = Vector3d(1,1,1)
glColor3f(*color)
x = self.waves_margin.left + 10.0 / 640 * self.width
y = self.height * 0.03
self._title_renderer.render(x, y)
def _set_gathered_color(self):
if self.download_completed():
self.gathered_color = self._fade_out_color()
self._gathered_segments_layer.refresh()
elif self.torrent_length > 0:
torrent_progress = float(self.gatherer.gathered_bytes()) / self.torrent_length
self.gathered_color = GATHERED_COLOR + (WAVEFORM_COLOR - GATHERED_COLOR) * pow(torrent_progress, 20)
else:
self.gathered_color = GATHERED_COLOR
def _fade_out_color(self):
time_after_completion = max(self.now - self.torrent_download_completion_time, 0)
if time_after_completion > FADE_OUT_DURATION:
return Vector3d(0,0,0)
else:
return WAVEFORM_COLOR * pow(1 - time_after_completion/FADE_OUT_DURATION, 0.15)
def active(self):
return len(self.playing_segments) > 0
def finished(self):
if self.download_completed():
time_after_completion = max(self.now - self.torrent_download_completion_time, 0)
if time_after_completion > FADE_OUT_DURATION:
return True
def draw_playing_segments(self):
glEnable(GL_LINE_SMOOTH)
glHint(GL_LINE_SMOOTH_HINT, GL_NICEST)
glEnable(GL_BLEND)
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
for segment in self.playing_segments.values():
segment.render()
def _render_gathered_segments(self):
glBegin(GL_QUADS)
x1 = self.waves_margin.left
x2 = self._waves_right
min_height = GATHERED_LINE_WIDTH * self._waves_height
for segment in self.gatherer.pieces():
y1 = self.byte_to_py(segment.torrent_begin)
y2 = max(self.byte_to_py(segment.torrent_end), y1 + min_height)
if (y2 - y1) > min_height:
d = min((y2 - y1) * 0.2, MAX_GRADIENT_HEIGHT * self._waves_height)
y1d = y1 + d
y2d = y2 - d
glColor3f(0, 0, 0)
glVertex2f(x1, y1)
glColor3f(*self.gathered_color)
glVertex2f(x1, y1d)
glVertex2f(x2, y1d)
glColor3f(0, 0, 0)
glVertex2f(x2, y1)
glColor3f(0, 0, 0)
glVertex2f(x1, y2)
glColor3f(*self.gathered_color)
glVertex2f(x1, y2d)
glVertex2f(x2, y2d)
glColor3f(0, 0, 0)
glVertex2f(x2, y2)
glColor3f(*self.gathered_color)
glVertex2f(x1, y1d)
glVertex2f(x1, y2d)
glVertex2f(x2, y2d)
glVertex2f(x2, y1d)
else:
glColor3f(0, 0, 0)
glVertex2f(x1, y1)
glColor3f(*self.gathered_color)
glVertex2f(x1, y2)
glVertex2f(x2, y2)
glColor3f(0, 0, 0)
glVertex2f(x2, y1)
glEnd()
def byte_to_py(self, byte):
return int(self.waves_margin.bottom + self.byte_to_relative_y(byte) * self._waves_height)
def byte_to_relative_y(self, byte):
return float(byte) / self.torrent_length
def handle_segment_waveform_value(self, segment, value):
segment.append_to_waveform(value)
