def getComments(self, posts_dict, csv_array): print(len(posts_dict), 'posts') for post_count, post_id in enumerate(posts_dict.keys()): resp = requests.get(self.url + '/comments/' + post_id + '.json?depth=1', headers={'User-agent': 'photoshopbot2'}) if resp.ok: resp_json = resp.json() print('Getting comments', post_count, '/', self.posts) for count, commment in enumerate( resp_json[1]['data']['children']): #print(count) if count >= self.comment_limit: # count starts at 0 break else: try: comm_img = commment['data']['body_html'].rsplit( '"')[3] if not (comm_img.endswith( ('.jpg', '.png', '.gif', 'jpeg'))): if comm_img[-1] == '/': pass else: filepath = ('./data/images/') filename = ('p_' + post_id + comm_img.rsplit("/", 1)[1]) os.makedirs(os.path.dirname(filepath + filename), exist_ok=True) try: Imgur.ImgurDownloader( comm_img, filepath, filename).save_images() csv_array.append([filename, 1]) except: count -= 1 pass else: image = requests.get( comm_img, allow_redirects=True, stream=True) #Request image filepath = ('./data/images/') filename = ('p_' + post_id + comm_img.rsplit("/", 1)[1]) os.makedirs(os.path.dirname(filepath + filename), exist_ok=True) open(filepath + filename, 'wb').write(image.content) csv_array.append([filename.split('.')[0], 1]) except (IndexError, KeyError): count -= 1 pass else: print(resp.reason) #self.makeCSV(csv_array) return csv_array
def search_img(song_name): print(song_name) print(type(song_name)) text = song_name df = pd.read_csv('temp/music_list.csv', index_col=False) img_url = df.loc[df['music_name'] == text]['music_sheet_url'].values[0] imgur_downloader.ImgurDownloader(imgur_url=img_url, file_name='song_img').save_images('image')
def search_img(self, song_name): text = song_name dbx.files_download_to_file('temp/music_list.csv', '/i_Piano/music_list.csv') df = pd.read_csv('temp/music_list.csv', index_col=False) img_url = df.loc[df['music_name'] == text]['music_sheet_url'].values[0] print(img_url) os.remove('temp/dst_image.png') imgur_downloader.ImgurDownloader( imgur_url=img_url, file_name='dst_image').save_images('temp')
def get_pictures(posts, path, dir_name_length): count = 0 for post in posts: if post.url[8:17] == "i.redd.it": count += download_image(post.url, path, dir_name_length) elif post.url[8:17] == "v.redd.it": continue # Maybe in a future version that supports downloading videos elif post.url[8:19] == "i.imgur.com": count += len( imgur.ImgurDownloader(post.url, path).save_images()[0]) return count
def getImages(url, search_filter, limit, posts_dict=None): if not posts_dict: # If no posts are passed, get post IDs and Images print('Getting Posts') resp = requests.get(url + '/' + search_filter + '/.json?count=' + str(limit), headers={'User-agent': 'photoshopbot2'}) if resp.ok: resp_json = resp.json() posts_dict = {} for post_number in range(0, limit): post_id = resp_json['data']['children'][post_number]['data'][ 'id'] # get ID post_img = resp_json['data']['children'][post_number]['data'][ 'url'] # get img link posts_dict[post_id] = post_img # add to Dict image = requests.get(post_img, allow_redirects=True, stream=True) #Request image filename = ('./data/original' + '/p_' + post_img.rsplit("/", 1)[1]) #filename = ('./data/Images/'+post_id+'/p_'+post_img.rsplit("/",1)[1]) os.makedirs(os.path.dirname(filename), exist_ok=True) open('./data/original' + '/p_' + post_img.rsplit("/", 1)[1], 'wb').write(image.content) #open('./data/Images/'+post_id+'/p_'+post_img.rsplit("/",1)[1], 'wb').write(image.content) #save image getImages(url, search_filter, limit, posts_dict) # Recursive call with post dicitonary else: print(resp.reason) else: # If posts are passed, get comment IDs and Images for each post print('Getting Comments') for post_id in posts_dict.keys(): resp = requests.get(url + '/comments/' + post_id + '.json?depth=1', headers={'User-agent': 'photoshopbot2'}) if resp.ok: resp_json = resp.json() comment_dict = {} for count, commment in enumerate( resp_json[1]['data']['children']): if count == 9: # count starts at 0 break else: comm_id = commment['data']['id'] try: comm_img = commment['data']['body_html'].rsplit( '"')[3] if not (comm_img.endswith( ('.jpg', '.png', '.gif', 'jpeg'))): if comm_img[-1] == '/': pass else: filename = ('./data/photoShopped/' + post_id + comm_img.rsplit("/", 1)[1]) #filename = ('./data/Images/'+post_id+'/commentImage/'+comm_img.rsplit("/",1)[1]) os.makedirs(os.path.dirname(filename), exist_ok=True) try: Imgur.ImgurDownloader( comm_img, './data/photoShopped/' ).save_images() #Imgur.ImgurDownloader(comm_img, './data/Images/'+post_id+'/commentImage').save_images() except: count -= 1 pass #print(comm_img.rsplit('rel="image_src" href=')[1]) else: image = requests.get( comm_img, allow_redirects=True, stream=True) #Request image filename = ('./data/photoShopped/' + post_id + comm_img.rsplit("/", 1)[1]) #filename = ('./data/Images/'+post_id+'/commentImage/'+comm_img.rsplit("/",1)[1]) os.makedirs(os.path.dirname(filename), exist_ok=True) open( './data/photoShopped/' + post_id + comm_img.rsplit("/", 1)[1], 'wb').write(image.content) #open('./data/Images/'+post_id+'/commentImage/'+comm_img.rsplit("/",1)[1], 'wb').write(image.content) #save image except IndexError: count -= 1 pass else: print(resp.reason)
def getImages(url, search_filter, limit, posts_dict=None): postArr = [] if not posts_dict: # If no posts are passed, get post IDs and Images print('Getting Posts') resp = requests.get(url + '/' + search_filter + '/.json?count=' + str(limit), headers={'User-agent': 'photoshopbot2'}) if resp.ok: resp_json = resp.json() posts_dict = {} print(len(resp_json['data']['children'])) print(resp_json['data']['after']) for post_number in range(0, limit): print(post_number) post_id = resp_json['data']['children'][post_number]['data'][ 'id'] # get ID post_img = resp_json['data']['children'][post_number]['data'][ 'url'] # get img link posts_dict[post_id] = post_img # add to Dict image = requests.get(post_img, allow_redirects=True, stream=True) #Request image filepath = ('./data/images/o_' + post_img.rsplit("/", 1)[1]) #filename = ('./data/Images/'+post_id+'/p_'+post_img.rsplit("/",1)[1]) os.makedirs(os.path.dirname(filepath), exist_ok=True) open('./data/images/o_' + post_img.rsplit("/", 1)[1], 'wb').write(image.content) postArr.append(['o_' + post_img.rsplit("/", 1)[1], 0]) #open('./data/Images/'+post_id+'/p_'+post_img.rsplit("/",1)[1], 'wb').write(image.content) #save image getImages(url, search_filter, limit, posts_dict) # Recursive call with post dicitonary else: print(resp.reason) else: # If posts are passed, get comment IDs and Images for each post print('Getting Comments') for post_id in posts_dict.keys(): resp = requests.get(url + '/comments/' + post_id + '.json?depth=1', headers={'User-agent': 'photoshopbot2'}) if resp.ok: resp_json = resp.json() for count, commment in enumerate( resp_json[1]['data']['children']): if count == 4: # count starts at 0 break else: comm_id = commment['data']['id'] try: comm_img = commment['data']['body_html'].rsplit( '"')[3] if not (comm_img.endswith( ('.jpg', '.png', '.gif', 'jpeg'))): if comm_img[-1] == '/': pass else: filepath = ('./data/images/') filename = ('p_' + post_id + comm_img.rsplit("/", 1)[1]) os.makedirs(os.path.dirname(filepath + filename), exist_ok=True) try: Imgur.ImgurDownloader( comm_img, filepath, filename).save_images() comment_arr.append([filename, 1]) except: count -= 1 pass else: image = requests.get( comm_img, allow_redirects=True, stream=True) #Request image filepath = ('./data/images/') filename = ('p_' + post_id + comm_img.rsplit("/", 1)[1]) os.makedirs(os.path.dirname(filepath + filename), exist_ok=True) open(filepath + filename, 'wb').write(image.content) comment_arr.append([filename, 1]) except IndexError: count -= 1 pass else: print(resp.reason) csv_arr = comment_arr + postArr #print(csv_arr,'\t end') with open("./data/data_labels.csv", "w+", newline='') as my_csv: csvWriter = csv.writer(my_csv, delimiter=',') csvWriter.writerows(csv_arr)