async def get_playlist_info(self, search: str):
await self.refresh_token()
async with Aiogoogle(user_creds=self.user_creds) as aiogoogle:
drive_v3 = await aiogoogle.discover('drive', 'v3')
folder_data = await aiogoogle.as_user(
drive_v3.files.get(
fileId=search,
fields=
'id,name,owners(displayName),createdTime,webViewLink',
supportsAllDrives=True))
async with Aiogoogle(user_creds=self.user_creds) as aiogoogle:
drive_v3 = await aiogoogle.discover('drive', 'v3')
file_data = await aiogoogle.as_user(
drive_v3.files.list(
q=f"mimeType contains 'audio' and '{search}' in parents",
fields='files(name,id),nextPageToken',
orderBy='folder,name,createdTime',
pageSize=1000,
supportsAllDrives=True,
includeItemsFromAllDrives=True))
if file_data is None:
raise GDriveError(
'Couldn\'t find anything that matches `{}`'.format(search))
data = {
"title": folder_data['name'],
"song_num": len(file_data['files'])
}
return data
async def decor(self, *args, **kwargs):
user_creds = kwargs.get('user_creds')
async with Aiogoogle(
client_creds=self.CLIENT_CREDS,
user_creds=user_creds
) as aiogoogle:
return await func(self, aiogoogle, *args, **kwargs)
async def get_playlist(self, search: str, include_name: bool = False):
await self.refresh_token()
async with Aiogoogle(user_creds=self.user_creds) as aiogoogle:
drive_v3 = await aiogoogle.discover('drive', 'v3')
data = await aiogoogle.as_user(
drive_v3.files.list(
q=f"mimeType contains 'audio' and '{search}' in parents",
fields='files(name,id),nextPageToken',
orderBy='folder,name,createdTime',
pageSize=1000,
supportsAllDrives=True,
includeItemsFromAllDrives=True))
if data is None:
raise GDriveError(
'Couldn\'t find anything that matches `{}`'.format(search))
sources = []
if not include_name:
for entry in data['files']:
sources.append(entry['id'])
else:
for entry in data['files']:
sources.append({"id": entry['id'], "name": entry['name']})
return sources
async def channel_info(self, channel_id: str):
# Get channel information
# Return a list
# Cases of success: [True, Channel_Id(str), Channel_Title(str), thumbnails_url(dict)]
while True:
try:
async with Aiogoogle(api_key=self.api_key) as google:
youtube = await google.discover('youtube', 'v3')
search_response = await google.as_api_key(
youtube.channels.list(
part='snippet',
id=channel_id
)
)
break
except TypeError:
raise TypeError
except:
pass
res = search_response.get('items', [])
if len(res) != 1: # channel search error
return [False]
else:
item = res[0]['snippet']
return [True, channel_id, item['title'], item['thumbnails']]
async def list_storage_buckets(project_id=creds["project_id"]):
async with Aiogoogle(service_account_creds=creds) as aiogoogle:
storage = await aiogoogle.discover("storage", "v1")
res = await aiogoogle.as_service_account(
storage.buckets.list(project=creds["project_id"])
)
pprint(res)
async def test_hardcoded_discovery_service_api_specs_is_latest_version():
CURRENT_VERSION = 1
async with Aiogoogle() as google:
discovery_apis_list = await google.list_api("discovery")
apis = discovery_apis_list["items"]
versions_available = [int(api["version"][-1:]) for api in apis]
for version in versions_available:
assert CURRENT_VERSION >= version
async def list_notes():
async with Aiogoogle(service_account_creds=creds) as aiogoogle:
async with aiohttp.ClientSession() as session:
async with session.get(DISCOVERY_DOC_URL) as resp:
disc_doc = await resp.json()
keep = GoogleAPI(disc_doc)
res = await aiogoogle.as_service_account(keep.notes.list())
pprint(res)
async def test_oauth2_manager_api_is_latest_version():
CURRENT_OAUTH2_API_VERSION = 2
async with Aiogoogle() as google:
oauth2_apis_list = await google.list_api("oauth2")
apis = oauth2_apis_list["items"]
versions_available = [int(api["version"][-1:]) for api in apis]
for version in versions_available:
assert CURRENT_OAUTH2_API_VERSION >= version
async def refresh_disc_docs_json():
file_errors = []
# Create new .data/ dir if one doesn't exists
this_folder = Path(__file__).parent
data_dir = this_folder / "data"
data_dir.mkdir(exist_ok=True)
async with ClientSession() as sess:
apis_pref = await sess.get(
"https://www.googleapis.com/discovery/v1/apis?preferred=true")
apis_pref = await apis_pref.json()
# Refresh all_apis in tests/tests_globals.py
all_apis = [(api["name"], api["version"]) for api in apis_pref["items"]]
all_apis = _pop_unstable_apis(all_apis)
final_all_apis = all_apis
async with Aiogoogle() as google:
tasks = [
google.discover(name, version) for (name, version) in all_apis
]
print('Requesting all APIs, this might take a while')
all_discovery_documents = await asyncio.gather(*tasks,
return_exceptions=True)
# Refresh discovery files in tests/data
for google_api, (name, version) in zip(all_discovery_documents, all_apis):
if isinstance(google_api, HTTPError):
e = google_api
if e.res.status_code != 404:
print('Non 404 error')
print('\033[91m\n' + e + '\n\033[0m')
else:
# only ignore if it's a 404 error. Should raise an error otherwise
final_all_apis = list(
filter(lambda api: (api[0] != name), final_all_apis))
file_errors.append({f"{name}-{version}": str(e)})
print(f'\033[91mError: Failed to download {name} {version}\033[0m')
continue
# Save discovery docuemnt as .json file to the newly created data dir
json_file = data_dir / f"{name}_{version}_discovery_doc.json"
with open(str(json_file), "w") as discovery_file:
json.dump(google_api.discovery_document, discovery_file)
print(f"saved {name}-{version} to {json_file}")
print("Done")
if file_errors:
print(f"Errors found: {str(file_errors)}")
with this_folder.joinpath("ALL_APIS.py").open("w") as f:
f.write("""### This file is autogenerated ###\n""")
f.write(f"ALL_APIS = {pprint.pformat(final_all_apis)}")
print("SUCCESS!")
async def verify_purchase(token, package_name, product_id):
async with Aiogoogle(service_account_creds=creds) as aiogoogle:
publisher_api = await aiogoogle.discover('androidpublisher', 'v3')
request = publisher_api.purchases.products.get(
token=token, productId=product_id, packageName=package_name)
validation_result = await aiogoogle.as_service_account(
request, full_res=True, raise_for_status=False)
pprint(validation_result.content)
async def get_video_json(self, id):
async with Aiogoogle(api_key=self.config_get('api_key')) as aiogoogle:
youtube_v3 = await aiogoogle.discover('youtube', 'v3')
request = youtube_v3.videos.list(
id=id, hl='en', part='snippet,contentDetails,statistics')
response = await aiogoogle.as_api_key(request)
if len(response['items']) == 0:
return None
else:
return response['items'][0]
async def find_folder(name: str, parent_id: str) -> dict:
aiogoogle = Aiogoogle(service_account_creds=creds)
async with aiogoogle:
drive_v3 = await aiogoogle.discover("drive", "v3")
result = await aiogoogle.as_service_account(
drive_v3.files.list(
q=f"mimeType='application/vnd.google-apps.folder' "
f"and name = '{name}' and parents in '{parent_id}'",
spaces="drive",
fields="files(id, name)"))
return result # {"files": [{"id": .., "name": ..}]}
async def loadasyncsheetservice(self, serviceAccountFile):
with open(serviceAccountFile, "r") as read_file:
SCOPES = ['https://www.googleapis.com/auth/spreadsheets']
credfile = json.load(read_file)
asynccreds = None
asynccreds = ServiceAccountCreds(scopes=SCOPES, **credfile)
async with Aiogoogle(
service_account_creds=asynccreds) as aiogoogle:
self._sheetServiceAsync = await aiogoogle.discover(
"sheets", "v4")
self._spreadsheetAsync = self._sheetServiceAsync.spreadsheets
self.aiogoogle = aiogoogle
async def create_folder(name: str, parent_id: Optional[str] = None) -> dict:
aiogoogle = Aiogoogle(service_account_creds=creds)
async with aiogoogle:
drive_v3 = await aiogoogle.discover("drive", "v3")
payload = {
"name": name,
"mimeType": "application/vnd.google-apps.folder"
}
if parent_id:
payload["parents"] = [parent_id]
result = await aiogoogle.as_service_account(
drive_v3.files.create(json=payload, fields="id"))
return result # {"id": ".. folder_id .."}
async def main():
aiogoogle = Aiogoogle(client_creds=CLIENT_CREDS)
uri = aiogoogle.oauth2.authorization_url(
client_creds=CLIENT_CREDS,
access_type="offline",
include_granted_scopes=True,
prompt="select_account",
)
webbrowser.open_new_tab(uri)
grant = input("Paste the code you received here, then press Enter")
full_user_creds = await aiogoogle.oauth2.build_user_creds(
grant=grant, client_creds=CLIENT_CREDS)
print(f"full_user_creds: {pprint.pformat(full_user_creds)}")
async def handler():
notion_events = get_notion_events()
async with Aiogoogle(
service_account_creds=service_account_creds) as google:
#connect to gcal api
gcal_api = await google.discover("calendar", "v3")
events_request = gcal_api.events.list(calendarId=gcal_id,
singleEvents=True,
timeMin=get_iso_timestamp())
events = await google.as_service_account(events_request)
async_tasks = []
#find all necessary tasks by matching notion to gcal events
for notion_event in notion_events:
gcal_match = [
x for x in events['items'] if x['id'] == notion_event['id']
]
new_gcal_event = format_notion_event_for_gcal(notion_event)
#there is a match!
if len(gcal_match) == 1:
gcal_match = gcal_match[0]
#check if data is the same
gcal_match = {
k: gcal_match[k]
for k in new_gcal_event if k in gcal_match
}
if not same_events(gcal_match, new_gcal_event):
print('updating event')
#update existing event in gcal
req = gcal_api.events.update(
calendarId=gcal_id,
eventId=gcal_match['id'],
json=format_notion_event_for_gcal(notion_event))
await google.as_service_account(req)
#no match
elif len(gcal_match) == 0:
print('creating new event')
#create event in gcal
req = gcal_api.events.insert(
calendarId=gcal_id,
json=format_notion_event_for_gcal(notion_event))
await google.as_service_account(req)
#find gcal events to delete
notion_ids = [x['id'] for x in notion_events]
deleted_events = [
x['id'] for x in events['items'] if x['id'] not in notion_ids
]
for old_event_id in deleted_events:
print('deleting event')
req = gcal_api.events.delete(calendarId=gcal_id,
eventId=old_event_id)
await google.as_service_account(req)
async def refresh_disc_docs_json():
file_errors = []
current_dir = os.getcwd()
# Create new .data/ dir if one doesn't exists
_check_for_correct_cwd(current_dir)
# Refresh all_apis in tests/tests_globals.py
all_apis = []
print("Refreshing all_apis in tests_globals.py")
async with ClientSession() as sess:
apis_pref = await sess.get(
"https://www.googleapis.com/discovery/v1/apis?preferred=true")
apis_pref = await apis_pref.json()
for api in apis_pref["items"]:
all_apis.append((api["name"], api["version"]))
all_apis = _pop_buggy_apis(_pop_unstable_apis(all_apis))
with open("tests/test_globals.py", "w") as f:
f.write("""### This file is autogenerated ###\n""")
f.write(f"ALL_APIS = {pprint.pformat(all_apis)}")
print("SUCCESS!")
# Refresh discovery files in tests/data
async with Aiogoogle() as aiogoogle:
for name, version in all_apis:
print(f"Downloading {name}-{version}")
try:
google_api = await aiogoogle.discover(name, version)
except Exception as e:
file_errors.append({f"{name}-{version}": str(e)})
continue
data_dir_name = current_dir + "/tests/data/"
try:
if not os.path.exists(data_dir_name):
os.makedirs(data_dir_name)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Save discovery docuemnt as .json file to the newly created data dir
file_name = (current_dir + "/tests/data/" + name + "_" + version +
"_discovery_doc.json")
with open(file_name, "w") as discovery_file:
json.dump(google_api.discovery_document, discovery_file)
print(f"saved {name}-{version} to {file_name}")
print("Done")
if file_errors:
print(f"Errors found: {str(file_errors)}")
async def create_source(self, search: str):
await self.refresh_token()
async with Aiogoogle(user_creds=self.user_creds) as aiogoogle:
drive_v3 = await aiogoogle.discover('drive', 'v3')
data = await aiogoogle.as_user(
drive_v3.files.get(
fileId=search,
fields=
'id,name,owners(displayName),createdTime,webViewLink',
supportsAllDrives=True))
sorted_info = await self.sort_info(data, search)
return sorted_info
async def upload_storage_bucket(bucket="test_bucket_clientlib",
file_name="one.csv",
file_path="/mnt/c/Users/Omar/Desktop/one.csv",
content_type="text/csv"):
async with Aiogoogle(service_account_creds=creds) as aiogoogle:
storage = await aiogoogle.discover("storage", "v1")
req = storage.objects.insert(
bucket=bucket,
name=file_name,
upload_file=file_path,
)
# You can autodetect mimetypes using python's built in `mimetypes` library
req.upload_file_content_type = content_type
res = await aiogoogle.as_service_account(req)
pprint(res)
async def _get_after(creds, gmail, timestamp):
async with Aiogoogle(user_creds=_get_user_creds(creds),
client_creds=_get_client_creds(creds)) as session:
print(f"Getting emails")
ids = await _get_email_ids(session, gmail)
print(f"Found {len(ids)} ids")
emails = await asyncio.gather(*(_get_email(session, gmail, key)
for key in ids))
print(f"Scraped {len(emails)} emails")
filtered = [email for email in emails if email[2] > timestamp]
print(f"Found {len(filtered)} new emails")
return filtered
async def refresh_token(self):
if self.refreshed:
time_passed = int(self.refreshed_time - time.time())
if time_passed < 2000:
return
async with Aiogoogle(user_creds=self.user_creds,
client_creds=self.client_creds) as aiogoogle:
creds = await aiogoogle.oauth2.refresh(self.user_creds,
self.client_creds)
creds['refresh_token'] = self.refreshtoken
self.user_creds = creds
self.refreshed_time = time.time()
self.refreshed = True
with open("token.json", 'w') as f:
json.dump(creds, f)
async def list_storage_buckets(project_id=creds["project_id"]):
aiogoogle = Aiogoogle(service_account_creds=creds)
await aiogoogle.service_account_manager.detect_default_creds_source()
async with aiogoogle:
storage = await aiogoogle.discover("storage", "v1")
res = await aiogoogle.as_service_account(
storage.buckets.list(project=creds["project_id"]))
# Doing it again just to show that you only need to call
# `detect_default_creds_source` once.
async with aiogoogle:
storage = await aiogoogle.discover("storage", "v1")
res = await aiogoogle.as_service_account(
storage.buckets.list(project=creds["project_id"]))
pprint(res)
async def main():
nonce = create_secret()
aiogoogle = Aiogoogle(client_creds=CLIENT_CREDS)
uri = aiogoogle.openid_connect.authorization_url(
client_creds=CLIENT_CREDS,
nonce=nonce,
access_type='offline',
include_granted_scopes=True,
prompt='select_account')
webbrowser.open_new_tab(uri)
grant = input('Paste the code you received here, then press Enter')
full_user_creds = await aiogoogle.openid_connect.build_user_creds(
grant=grant, client_creds=CLIENT_CREDS, nonce=nonce, verify=False)
full_user_info = await aiogoogle.openid_connect.get_user_info(
full_user_creds)
print(
f"full_user_creds: {pprint.pformat(full_user_creds)}\n\nfull_user_info: {pprint.pformat(full_user_info)}"
)
async def stream_check(self, channel_id: str, channel_title: str):
live_url = f'https://www.youtube.com/channel/{channel_id}/live'
while True:
try:
async with aiohttp.ClientSession() as session:
async with session.get(live_url, proxy=self.proxy) as resp:
c = await resp.text()
break
except:
pass
c = str(c)
if "offline" not in c: # live now
st = c.find('"video_id":"') + 12
ed = st
while c[ed] != '"':
ed += 1
video_id = c[st:ed]
if video_id[-1] == '=':
return [False]
print(f"{channel_title} is online")
while True:
try:
async with Aiogoogle(api_key=self.api_key) as google:
youtube = await google.discover('youtube', 'v3')
search_response = await google.as_api_key(
youtube.videos.list(
part='snippet',
id=video_id
)
)
break
except TypeError:
raise TypeError
except:
pass
res = search_response.get('items', [])
if len(res) == 1:
res = res[0]['snippet']
return [True, res['channelTitle'], res['title'], res['thumbnails']]
else:
return [False]
else: # offline now
return [False]
async def main():
if not USER_CREDS:
raise Exception(
f"ERROR: user_creds={USER_CREDS}, call get_token first")
db = db_connect()
await on_init(db)
service = await get_service()
while True:
try:
start_time = time.time()
ids = await get_ids(db)
print(f"...{i} Batch request ids count={len(ids)}")
req_list = get_batch_requests_list(service, ids)
req_tasks = [
get_batch_response(
Aiogoogle(user_creds=USER_CREDS,
client_creds=CLIENT_CREDS), r) for r in req_list
]
resp_list = await asyncio.gather(*req_tasks,
return_exceptions=True)
resp_tasks = [save_batch_response(db, r) for r in resp_list]
await asyncio.gather(*resp_tasks, return_exceptions=True)
await show_stats(db)
exec_time = time.time() - start_time
print(f"...pause. Execution time: {exec_time}")
await asyncio.sleep(3)
except Exception as e:
print(f"...Error {str(e)}")
continue
print("...Finished...")
async def uploaded_video_list(self, channel_id: str):
while True:
try:
async with Aiogoogle(api_key=self.api_key) as google:
youtube = await google.discover('youtube', 'v3')
search_response = await google.as_api_key(
youtube.channels.list(
part='contentDetails',
id=channel_id
)
)
break
except TypeError:
raise TypeError
except:
pass
res = search_response.get('items', [])
playlist = res[0]['contentDetails']['relatedPlaylists']['uploads']
return playlist
async def video_list(self, playlist, maxresult=5):
while True:
try:
async with Aiogoogle(api_key=self.api_key) as google:
youtube = await google.discover('youtube', 'v3')
search_response = await google.as_api_key(
youtube.playlistItems.list(
part='snippet',
playlistId=playlist,
maxResults=maxresult
)
)
break
except TypeError:
raise TypeError
except:
pass
res = search_response.get('items', [])
return res
async def ready_download(self, data: dict):
INFO(f"Started downloading {data.title} from {data.search}")
await self.refresh_token()
while not os.path.isfile(f"audio_cache\\{data.expected_filename}"):
async with Aiogoogle(user_creds=self.user_creds) as aiogoogle:
drive_v3 = await aiogoogle.discover('drive', 'v3')
try:
await aiogoogle.as_user(
drive_v3.files.get(
fileId=data.search,
download_file=
f"audio_cache\\{data.expected_filename}",
alt="media"))
except Exception as e:
if str(e) == "Line is too long":
pass
else:
INFO(e)
INFO(f"Downloaded {data.title}")
try:
tags = MP3(f"audio_cache\\{data.expected_filename}")
except:
return data
if not os.path.isfile(f"image_cache\\{data.title}.jpg"):
try:
pic_key = [key for key in list(tags.keys())
if "APIC" in key][0]
pic = tags.get(pic_key)
im = Image.open(BytesIO(pic.data))
im.save(f"image_cache\\{data.title}.jpg")
except:
data.thumbnail = "https://webrandum.net/mskz/wp-content/uploads/pz-linkcard/cache/7232681e168b08a699569b8291bbeaa3c0435198368ccf2b11fa8cca02e5e115"
try:
data.artist = tags.get('TPE1').text[0]
except:
pass
data.duration = int(tags.info.length)
return data
async def channel_statistics(self, channel_id: str):
# 'viewCount', 'commentCount', 'subscriberCount', 'hiddenSubscriberCount', 'videoCount'
while True:
try:
async with Aiogoogle(api_key=self.api_key) as google:
youtube = await google.discover('youtube', 'v3')
search_response = await google.as_api_key(
youtube.channels.list(
part='statistics',
id=channel_id
)
)
break
except TypeError:
raise TypeError
except:
pass
res = search_response.get('items', [])
assert len(res) == 1
return res[0]['statistics']
async def channel_search(self, Vtuber_Name: str):
# Search ytb channel by name
# Return a list
# Cases of success: [True, Channel_Id(str), Channel_Title(str), thumbnails_url(dict)]
while True:
try:
async with Aiogoogle(api_key=self.api_key) as google:
youtube = await google.discover('youtube', 'v3')
search_response = await google.as_api_key(
youtube.search.list(
part='snippet',
q=Vtuber_Name,
maxResults=10
)
)
break
except TypeError:
raise TypeError
except:
pass
res = search_response.get('items', [])
if len(res) == 0: # no such Vtuber
return [False]
else:
x = {}
for item in res:
id = item['snippet']['channelId']
if id in x:
x[id][1] += 1
else:
x[id] = [item['snippet']['channelTitle'], 1]
most_possible_channel_id = res[0]['snippet']['channelId']
for possible_channel_id in x:
if x[possible_channel_id][1] > x[most_possible_channel_id][1]:
most_possible_channel_id = possible_channel_id
print(f'{x[most_possible_channel_id][0]} selected')
ch_info = await self.channel_info(most_possible_channel_id)
return ch_info