def retrieve_and_store_user_infos() -> None:
global __runs
__runs += 1
util.prnt(f'----------')
util.prnt(f'Starting data collection run {__runs}')
utc_now = util.get_utc_now()
data = collect_data(utc_now)
data_file_name = util.get_collect_file_name(utc_now)
if settings.SETTINGS['store_at_filesystem']:
util.dump_data(data, data_file_name, settings.DEFAULT_COLLECT_FOLDER)
if settings.SETTINGS['store_at_gdrive']:
tries_left = 5
while tries_left > 0:
tries_left -= 1
try:
gdrive.upload_file(data, data_file_name)
tries_left = 0
except Exception as error:
if tries_left > 0:
gdrive.init(force=True)
else:
raise error
def save_to_gdrive(content: str, file_name: str) -> None:
gdrive.init(force=True)
try:
gdrive.upload_file(file_name, content)
except Exception as error:
err(f'Could not upload the file to google drive', error)
else:
prnt(f'Successfully uploaded {file_name} to google drive.')
def print_quota():
gdrive.init()
about = gdrive.get_about()
name = about.get('name', '-')
max_quota = int(about.get('quotaBytesTotal', -1))
used_quota = int(about.get('quotaBytesUsed', -1))
trash_quota = int(about.get('quotaBytesUsedInTrash', -1))
combined_quota = used_quota + trash_quota
combined_quota = combined_quota if combined_quota > -2 else -1
available_quota = max_quota - combined_quota
available_quota_percentage = (1 - (combined_quota / max_quota))
util.prnt(f'Quotas for account: {name}')
util.prnt(
f'Max Quota: {max_quota} bytes ({util.convert_to_bytes_count(max_quota)})'
)
util.prnt(
f'Used Quota: {used_quota} bytes ({util.convert_to_bytes_count(used_quota)})'
)
util.prnt(
f'Trashed Quota: {trash_quota} bytes ({util.convert_to_bytes_count(trash_quota)})'
)
util.prnt(
f'Available Quota: {available_quota} bytes ({util.convert_to_bytes_count(available_quota)}, {available_quota_percentage * 100:.2f} %)'
)
def init(store_at_filesystem: bool = None,
store_at_gdrive: bool = None,
verbose: bool = None,
no_time: bool = None) -> dict:
PWD = os.getcwd()
sys.path.insert(0, f'{PWD}/')
if verbose is not None:
settings.SETTINGS['print_verbose'] = verbose
if no_time is None:
util.vrbs(f'Print timestamps: {settings.SETTINGS["print_timestamps"]}')
else:
print_timestamps = not no_time
settings.SETTINGS['print_timestamps'] = print_timestamps
util.vrbs(f'Print timestamps: {print_timestamps}')
if store_at_filesystem is None:
util.vrbs(
f'Store at filesystem: {settings.SETTINGS["store_at_fileystem"]}')
else:
settings.SETTINGS['store_at_filesystem'] = store_at_filesystem
util.vrbs(f'Store at filesystem: {store_at_filesystem}')
if store_at_gdrive is None:
util.vrbs(
f'Store at google drive: {settings.SETTINGS["store_at_gdrive"]}')
else:
settings.SETTINGS['store_at_gdrive'] = store_at_gdrive
util.vrbs(f'Store at google drive: {store_at_gdrive}')
if store_at_gdrive:
gdrive.init()
for folder_name in settings.CREATE_FOLDERS_ON_COLLECT:
if not os.path.isdir(folder_name):
os.mkdir(folder_name)
def update_settings(handler, **kwargs):
gae.update_settings(**kwargs)
_settings = gae.get_settings()
gdrive.init(_settings.gdrive_dev_key, _settings.gdrive_wiki_id)
data.init(_settings.github_key, _settings.github_repo)
from googleapiclient import discovery
from oauth2client import client
from oauth2client.contrib import appengine
from google.appengine.api import memcache
#import github as data
MIME_TYPES = {'.css': 'text/css', '.js': 'application/javascript' }
JINJA_ENVIRONMENT = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
autoescape=True,
extensions=['jinja2.ext.autoescape'])
data = gae.NullData()
_settings = gae.get_settings()
gdrive.init(_settings.gdrive_dev_key, _settings.gdrive_wiki_id)
data.init(_settings.github_key, _settings.github_repo)
app = webapp2.WSGIApplication([(gdrive.decorator.callback_path, gdrive.decorator.callback_handler())])
def extract_params(f):
def wrapper(handler, *args, **kwargs):
kwargs.update(handler.request.params)
if handler.request.headers.get('content-type') == 'application/json':
kwargs.update(json.loads(handler.request.body))
res = f(handler, *args, **kwargs)
if res:
res = json.dumps(res) if isinstance(res, dict) else str(res)
handler.response.write(res)
return wrapper
def get_about():
gdrive.init()
about = gdrive.get_about()
util.print_dict(about)
def clear_trash_bin():
gdrive.init()
files = gdrive.get_trashed_files()
for file in files:
file.Delete()
def delete_all_files_but_latest_daily(from_year: int = None,
from_month: int = None,
from_day: int = None,
to_year: int = None,
to_month: int = None,
to_day: int = None,
delete_tourney_data: bool = False):
gdrive.init()
utc_now = util.get_utc_now()
if not from_year and not from_month and not from_day:
from_timestamp = datetime(2019,
10,
9,
tzinfo=settings.DEFAULT_TIMEZONE)
else:
from_year = from_year or 2019
from_month = from_month or (10 if from_year == 2019 else 1)
from_day = from_day or (9 if from_year == 2019 and from_month == 10
else 1)
from_timestamp = datetime(from_year,
from_month,
from_day,
tzinfo=settings.DEFAULT_TIMEZONE)
to_year = to_year or utc_now.year
to_month = to_month or utc_now.month
to_day = to_day or utc_now.day
to_timestamp = datetime(
to_year, to_month, to_day,
tzinfo=settings.DEFAULT_TIMEZONE) + util.ONE_DAY
while from_timestamp < to_timestamp:
file_name_prefix = f'{settings.FILE_NAME_COLLECT_PREFIX}{from_timestamp.year:02d}{from_timestamp.month:02d}{from_timestamp.day:02d}'
util.prnt(
f'Checking files for deletion with prefix: {file_name_prefix}')
if delete_tourney_data or not util.is_tourney_running(
utc_now=from_timestamp):
file_list = gdrive.get_files_in_folder(settings.GDRIVE_FOLDER_ID,
file_name_prefix)
if file_list:
file_count = len(file_list)
if file_count > 1:
util.prnt(f'Found {len(file_list)} files.')
else:
util.prnt(f'Already clean.')
file_list = sorted(file_list, key=lambda f: f['title'])
file_list.pop()
else:
util.prnt(f'No such files found.')
for file in file_list:
skip = None
while not skip:
tries = 0
while tries <= 2:
util.prnt(
f'Permanently deleting file: {file["title"]}')
if gdrive.try_delete_file(file):
skip = True
break
tries += 1 # Attempt to delete 3 times
if tries == 3:
if skip is None: # If deletion fails 3 times, attempt to re-initialize the gdrive client
gdrive.init(force=True)
skip = False
else:
skip = True # If re-initializing the gdrive client doesn't help, skip the file
from_timestamp += util.ONE_DAY