class Storage: def __init__(self, mode=StorageModes.MEMORY): self.mode = mode if self.mode == StorageModes.PERSISTENT: self.cache = SqliteDict('../my_db.sqlite', autocommit=True) elif self.mode == StorageModes.MEMORY: self.cache = dict() def set(self, k, v): self.cache[k] = v if self.mode == StorageModes.PERSISTENT: # need to commit manually, as autocomit only commits with commit(blocking=False) and might not persist data self.cache.commit() def dump(self, k): self.cache.pop(k) if self.mode == StorageModes.PERSISTENT: # need to commit manually, as autocomit only commits with commit(blocking=False) and might not persist data self.cache.commit() def get(self, k): return self.cache.get(k) def append(self, k, v): current_data = self.cache.get(k) if not current_data: self.set(k, [v]) else: if not isinstance(current_data, list): current_data = [current_data] current_data.append(v) self.set(k, current_data)
class GoogleDrive: auth_url = 'https://accounts.google.com/o/oauth2/v2/auth' token_url = 'https://www.googleapis.com/oauth2/v4/token' api_url = 'https://www.googleapis.com/drive/' redirect_url = 'urn:ietf:wg:oauth:2.0:oob' scopes = ['https://www.googleapis.com/auth/drive.readonly'] def __init__(self, config, client_id: str, client_secret: str, token_path: str, cache_path: str): self.cfg = config self.client_id = client_id self.client_secret = client_secret self.token_path = token_path self.cache_path = cache_path self.cache = SqliteDict(self.cache_path, tablename='cache', encode=json.dumps, decode=json.loads, autocommit=False) self.transcodes_cache = ExpiringDict(max_len=5000, max_age_seconds=2 * (60 * 60)) self.token = self._load_token() self.token_refresh_lock = Lock() self.http = self._new_http_object() ############################################################ # CORE CLASS METHODS ############################################################ def get_auth_link(self): auth_url, state = self.http.authorization_url(self.auth_url, access_type='offline', prompt='select_account') return auth_url def exchange_code(self, code: str): token = self.http.fetch_token(self.token_url, code=code, client_secret=self.client_secret) if 'access_token' in token: self._token_saver(token) return self.token def query(self, path: str, method: str = 'GET', fetch_all_pages: bool = False, callbacks={}, **kwargs): resp: Response = None pages: int = 1 resp_json = {} request_url = self.api_url + path.lstrip('/') if not path.startswith('http') else path try: while True: resp = self._do_query(request_url, method, **kwargs) log.debug(f"Request URL: {resp.url}") log.debug(f"Request ARG: {kwargs}") log.debug(f'Response Status: {resp.status_code} {resp.reason}') if 'stream' in kwargs and kwargs['stream']: return True, resp, None if 'Content-Type' in resp.headers and 'json' in resp.headers['Content-Type']: if fetch_all_pages: resp_json.pop('nextPageToken', None) new_json = resp.json() # does this page have changes extended_changes = False changes = [] if 'changes' in new_json: if 'changes' in resp_json: changes.extend(resp_json['changes']) changes.extend(new_json['changes']) extended_changes = True resp_json.update(new_json) if extended_changes: resp_json['changes'] = changes else: return False if resp.status_code != 200 else True, resp, resp.text # call page_token_callback to update cached page_token, if specified if 'page_token_callback' in callbacks: if 'nextPageToken' in resp_json: callbacks['page_token_callback'](resp_json['nextPageToken']) elif 'newStartPageToken' in resp_json: callbacks['page_token_callback'](resp_json['newStartPageToken']) # call data_callback, fetch_all_pages is true if fetch_all_pages and 'data_callback' in callbacks: callbacks['data_callback'](resp.json(), callbacks) # handle nextPageToken if fetch_all_pages and 'nextPageToken' in resp_json and resp_json['nextPageToken']: # there are more pages pages += 1 log.info("Fetching extra results from page %d", pages) if 'params' in kwargs: kwargs['params'].update({'pageToken': resp_json['nextPageToken']}) elif 'json' in kwargs: kwargs['json'].update({'pageToken': resp_json['nextPageToken']}) elif 'data' in kwargs: kwargs['data'].update({'pageToken': resp_json['nextPageToken']}) continue break return True if resp_json and len(resp_json) else False, resp, resp_json if ( resp_json and len(resp_json)) else resp.text except Exception: log.exception(f"Exception sending request to {request_url} with kwargs={kwargs}: ") return False, resp, None ############################################################ # DRIVE FUNCTIONS ############################################################ def validate_access_token(self): success, resp, data = self.query('/v3/changes/startPageToken', params={'supportsTeamDrives': self.cfg.google.teamdrive}) if success and resp.status_code == 200: if 'startPageToken' not in data: log.error("Failed validate up to date access_token:\n\n%s\n", data) return False return True else: log.error("Error validating access token, status_code = %d, data =\n\n%s\n", resp.status_code if resp is not None else 0, data) return False def get_changes(self, new_items_callback=None, removed_items_callback=None): callbacks = {'page_token_callback': self._page_token_saver, 'data_callback': self._process_changes} if new_items_callback: callbacks['new_items_callback'] = new_items_callback if removed_items_callback: callbacks['removed_items_callback'] = removed_items_callback success, resp, data = self.query('/v3/changes', params={ 'pageToken': self.token['page_token'] if 'page_token' in self.token else '1', 'pageSize': 1000, 'includeRemoved': True, 'includeTeamDriveItems': self.cfg.google.teamdrive, 'supportsTeamDrives': self.cfg.google.teamdrive, 'fields': 'changes(file(md5Checksum,mimeType,modifiedTime,' 'name,parents,teamDriveId,trashed),' 'fileId,removed,teamDrive(id,name),' 'teamDriveId),newStartPageToken,nextPageToken'}, fetch_all_pages=True, callbacks=callbacks) return def get_file(self, file_id, stream=True, headers=None, timeout=30): req_url = '/v2/files/%s' % file_id if not file_id.startswith('http') else file_id success, resp, data = self.query(req_url, params={ 'includeTeamDriveItems': self.cfg.google.teamdrive, 'supportsTeamDrives': self.cfg.google.teamdrive, 'alt': 'media' }, stream=stream, headers=headers, timeout=timeout) return resp def get_stream_link(self, file_id): # validate / refersh current access_token if not self.validate_access_token(): return '' log.debug("Validated access_token is current") # generate url req = Request('GET', f'{self.api_url.rstrip("/")}/v2/files/{file_id}', params={'includeTeamDriveItems': self.cfg.google.teamdrive, 'supportsTeamDrives': self.cfg.google.teamdrive, 'alt': 'media', 'access_token': self.token['access_token']}).prepare() log.debug(f'Direct Stream URL: {req.url}') return req.url def get_transcodes(self, file_id): # do we have the transcoded versions already cached within the last 5 minutes? cached_transcodes = self.transcodes_cache.get(file_id, None) if cached_transcodes is not None and len(cached_transcodes): log.debug(f"Loaded {len(cached_transcodes)} transcode streams from temporary cache for: {file_id}") return cached_transcodes # retrieve transcoded versions from google docs success, resp, data = self.query(f'https://docs.google.com/get_video_info?docid={file_id}') if not success or (not data or 'fmt_stream_map' not in data or 'fmt_list' not in data): log.error(f"Failed to find transcoded versions data for: {file_id}") return None # parse main response tmp = parse_qs(data) tmp_versions = tmp['fmt_list'][0] tmp_stream_map = tmp['fmt_stream_map'][0] drive_stream_cookie = resp.cookies.get('DRIVE_STREAM', '') # parse required variables transcode_versions = {} transcode_streams = {} # parse version list for version in tmp_versions.split(','): tmp_v = version.split('/') transcode_versions[tmp_v[0]] = tmp_v[1].split('x')[1] if not len(transcode_versions): log.error(f"Failed to parse transcoded versions (fmt_list) for: {file_id}") return None # parse transcode lists for stream in tmp_stream_map.split(','): tmp_s = stream.split('|') transcode_streams[transcode_versions[tmp_s[0]]] = tmp_s[1] if not len(transcode_streams): log.error(f"Failed to parse transcoded streams (fmt_stream_map) for: {file_id}") return None # cache the transcode streams for 5 minutes self.transcodes_cache[file_id] = transcode_streams log.debug(f"Added {len(transcode_streams)} transcode streams to temporary cache for: {file_id}") return transcode_streams ############################################################ # CACHE ############################################################ def get_id_metadata(self, item_id, teamdrive_id=None): # return cache from metadata if available cached_metadata = self._get_cached_metdata(item_id) if cached_metadata: return True, cached_metadata # does item_id match teamdrive_id? if teamdrive_id is not None and item_id == teamdrive_id: success, resp, data = self.query('v3/teamdrives/%s' % str(item_id)) if success and resp.status_code == 200 and 'name' in data: # we successfully retrieved this teamdrive info, lets place a mimeType key in the result # so we know it needs to be cached data['mimeType'] = 'application/vnd.google-apps.folder' else: # retrieve file metadata success, resp, data = self.query('v3/files/%s' % str(item_id), params={ 'supportsTeamDrives': self.cfg.google.teamdrive, 'fields': 'id,md5Checksum,mimeType,modifiedTime,name,parents,' 'trashed,teamDriveId'}) if success and resp.status_code == 200: return True, data else: log.error("Error retrieving metadata for item %r:\n\n%s\n", item_id, data) return False, data def get_id_file_paths(self, item_id, teamdrive_id=None): file_paths = [] added_to_cache = 0 try: def get_item_paths(obj_id, path, paths, new_cache_entries, teamdrive_id=None): success, obj = self.get_id_metadata(obj_id, teamdrive_id) if not success: return new_cache_entries teamdrive_id = teamdrive_id if 'teamDriveId' not in obj else obj['teamDriveId'] # add item object to cache if we know its not from cache if 'mimeType' in obj: # we know this is a new item fetched from the api, because the cache does not store this field self.add_item_to_cache(obj['id'], obj['name'], [] if 'parents' not in obj else obj['parents']) new_cache_entries += 1 if path.strip() == '': path = obj['name'] else: path = os.path.join(obj['name'], path) if 'parents' in obj and obj['parents']: for parent in obj['parents']: new_cache_entries += get_item_paths(parent, path, paths, new_cache_entries, teamdrive_id) if (not obj or 'parents' not in obj or not obj['parents']) and len(path): paths.append(path) return new_cache_entries return new_cache_entries added_to_cache += get_item_paths(item_id, '', file_paths, added_to_cache, teamdrive_id) if added_to_cache: log.debug("Dumping cache due to new entries!") self._dump_cache() if len(file_paths): return True, file_paths else: return False, file_paths except Exception: log.exception("Exception retrieving filepaths for '%s': ", item_id) return False, [] def add_item_to_cache(self, item_id, item_name, item_parents): if item_id not in self.cache: log.debug("Added '%s' to cache: %s", item_id, item_name) self.cache[item_id] = {'name': item_name, 'parents': item_parents} return def remove_item_from_cache(self, item_id): if self.cache.pop(item_id, None): return True return False def get_item_name_from_cache(self, item_id): try: item = self.cache.get(item_id) return item['name'] if isinstance(item, dict) else 'Unknown' except Exception: pass return 'Unknown' def get_item_from_cache(self, item_id): try: item = self.cache.get(item_id, None) return item except Exception: pass return None ############################################################ # INTERNALS ############################################################ def _do_query(self, request_url: str, method: str, **kwargs): tries: int = 0 max_tries: int = 2 lock_acquirer: bool = False resp: Response = None use_timeout: int = 30 # override default timeout if 'timeout' in kwargs and isinstance(kwargs['timeout'], int): use_timeout = kwargs['timeout'] kwargs.pop('timeout', None) # remove un-needed kwargs kwargs.pop('fetch_all_pages', None) kwargs.pop('page_token_callback', None) # do query while tries < max_tries: if self.token_refresh_lock.locked() and not lock_acquirer: log.debug("Token refresh lock is currently acquired... trying again in 500ms") time.sleep(0.5) continue if method == 'POST': resp = self.http.post(request_url, timeout=use_timeout, **kwargs) elif method == 'PATCH': resp = self.http.patch(request_url, timeout=use_timeout, **kwargs) elif method == 'DELETE': resp = self.http.delete(request_url, timeout=use_timeout, **kwargs) else: resp = self.http.get(request_url, timeout=use_timeout, **kwargs) tries += 1 if resp.status_code == 401 and tries < max_tries: # unauthorized error, lets refresh token and retry self.token_refresh_lock.acquire(False) lock_acquirer = True log.warning(f"Unauthorized Response (Attempts {tries}/{max_tries})") self.token['expires_at'] = time() - 10 self.http = self._new_http_object() else: break return resp def _load_token(self): try: if not os.path.exists(self.token_path): return {} with open(self.token_path, 'r') as fp: return json.load(fp) except Exception: log.exception(f"Exception loading token from {self.token_path}: ") return {} def _dump_token(self): try: with open(self.token_path, 'w') as fp: json.dump(self.token, fp, indent=2) return True except Exception: log.exception(f"Exception dumping token to {self.token_path}: ") return False def _token_saver(self, token: dict): # update internal token dict self.token.update(token) try: if self.token_refresh_lock.locked(): self.token_refresh_lock.release() except Exception: log.exception("Exception releasing token_refresh_lock: ") self._dump_token() log.info("Renewed access token!") return def _page_token_saver(self, page_token: str): # update internal token dict self.token['page_token'] = page_token self._dump_token() return def _new_http_object(self): return OAuth2Session(client_id=self.client_id, redirect_uri=self.redirect_url, scope=self.scopes, auto_refresh_url=self.token_url, auto_refresh_kwargs={'client_id': self.client_id, 'client_secret': self.client_secret}, token_updater=self._token_saver, token=self.token) def _get_cached_metdata(self, item_id): if item_id in self.cache: return self.cache[item_id] return None def _dump_cache(self): self.cache.commit() return def _remove_unwanted_paths(self, paths_list: list, mime_type: str): # remove paths that were not allowed - this is always enabled for item_path in copy(paths_list): allowed_path = False for allowed_file_path in self.cfg.google.allowed.file_paths: if item_path.lower().startswith(allowed_file_path.lower()): allowed_path = True break if not allowed_path: log.debug("Ignoring %r because its not an allowed path", item_path) paths_list.remove(item_path) continue # remove unallowed extensions if self.cfg.google.allowed.file_extensions: for item_path in copy(paths_list): allowed_file = False for allowed_extension in self.cfg.google.allowed.file_extensions_list: if item_path.lower().endswith(allowed_extension.lower()): allowed_file = True break if not allowed_file: log.debug("Ignoring %r because it was not an allowed extension", item_path) paths_list.remove(item_path) # remove unallowed mimes if self.cfg.google.allowed.mime_types: allowed_file = False for allowed_mime in self.cfg.google.allowed.mime_types_list: if allowed_mime.lower() in mime_type.lower(): if 'video' in mime_type.lower(): # we want to validate this is not a .sub file, which for some reason, google shows as video/MP2G double_checked_allowed = True for item_path in paths_list: if item_path.lower().endswith('.sub'): double_checked_allowed = False if double_checked_allowed: allowed_file = True break else: allowed_file = True break if not allowed_file: log.debug("Ignoring %s because it was not an allowed mime: %s", paths_list, mime_type) for item_path in copy(paths_list): paths_list.remove(item_path) def _process_changes(self, data: dict, callbacks: dict = {}): removed_file_paths = {} added_file_paths = {} if not data or 'changes' not in data: log.error("There were no changes to process") return log.info("Processing %d changes", len(data['changes'])) # process changes for change in data['changes']: if 'file' in change and 'fileId' in change: # dont consider trashed/removed events for processing if ('trashed' in change['file'] and change['file']['trashed']) or ( 'removed' in change and change['removed']): # store the removed file paths - only if we have this item cached, otherwise we are not interested # as we would not have stored it anyway... item_exists = self.get_item_from_cache(change['fileId']) if item_exists is not None: success, item_paths = self.get_id_file_paths(change['fileId'], change['file']['teamDriveId'] if 'teamDriveId' in change['file'] else None) self._remove_unwanted_paths(item_paths, change['file']['mimeType'] if 'mimeType' in change[ 'file'] else 'Unknown') if success and len(item_paths): if change['fileId'] in removed_file_paths: removed_file_paths[change['fileId']].extend(item_paths) else: removed_file_paths[change['fileId']] = item_paths # remove item from cache if self.remove_item_from_cache(change['fileId']): log.debug("Removed '%s' from cache: %s", change['fileId'], change['file']['name']) continue existing_cache_item = self.get_item_from_cache(change['fileId']) existing_success, existing_cache_item_paths = self.get_id_file_paths(change['fileId'], change['file']['teamDriveId'] if 'teamDriveId' in change[ 'file'] else None) if \ existing_cache_item is not None else (None, None) # we always want to add changes to the cache so renames etc can be reflected inside the cache self.add_item_to_cache(change['fileId'], change['file']['name'], [] if 'parents' not in change['file'] else change['file']['parents']) # dont process folder events if 'mimeType' in change['file'] and 'vnd.google-apps.folder' in change['file']['mimeType']: # ignore this change as we dont want to scan folders continue # get this files paths success, item_paths = self.get_id_file_paths(change['fileId'], change['file']['teamDriveId'] if 'teamDriveId' in change[ 'file'] else None) # remove unwanted paths if existing_success and len(existing_cache_item_paths): self._remove_unwanted_paths(existing_cache_item_paths, change['file']['mimeType'] if 'mimeType' in change[ 'file'] else 'Unknown') if success and len(item_paths): self._remove_unwanted_paths(item_paths, change['file']['mimeType'] if 'mimeType' in change[ 'file'] else 'Unknown') # was this an existing item? if (existing_cache_item is not None and existing_success and len(existing_cache_item_paths)) and ( success and len(item_paths)): # this was an existing item, and we are re-processing it again # we need to find the differences between the before and after paths. existing_path_set = set(existing_cache_item_paths) new_path_set = set(item_paths) removed_item_paths = existing_path_set.difference(new_path_set) added_item_paths = new_path_set.difference(existing_path_set) if len(removed_item_paths): if change['fileId'] in removed_file_paths: removed_file_paths[change['fileId']].extend(list(removed_item_paths)) else: removed_file_paths[change['fileId']] = list(removed_item_paths) if len(added_item_paths): if change['fileId'] in added_file_paths: added_file_paths[change['fileId']].extend(list(added_item_paths)) else: added_file_paths[change['fileId']] = list(added_item_paths) elif success and len(item_paths): # these are new paths/files that were not already in the cache if change['fileId'] in added_file_paths: added_file_paths[change['fileId']].extend(item_paths) else: added_file_paths[change['fileId']] = item_paths elif 'teamDrive' in change and 'teamDriveId' in change: # this is a teamdrive change # dont consider trashed/removed events for processing if 'removed' in change and change['removed']: # remove item from cache if self.remove_item_from_cache(change['teamDriveId']): log.info("Removed teamDrive '%s' from cache: %s", change['teamDriveId'], change['teamDrive']['name'] if 'name' in change[ 'teamDrive'] else 'Unknown teamDrive') continue if 'id' in change['teamDrive'] and 'name' in change['teamDrive']: # we always want to add changes to the cache so renames etc can be reflected inside the cache self.add_item_to_cache(change['teamDrive']['id'], change['teamDrive']['name'], []) continue # always dump the cache after running changes self._dump_cache() log.info('%d added / %d removed', len(added_file_paths), len(removed_file_paths)) # call further callbacks if len(removed_file_paths) and 'removed_items_callback' in callbacks: callbacks['removed_items_callback'](removed_file_paths) if len(added_file_paths) and 'new_items_callback' in callbacks: callbacks['new_items_callback'](added_file_paths) return
class DropNotClient(Thread): def __init__(self, sync_dir, target, port): """ Initialise the DropNot Client :param sync_dir: Directory to synchronise on the client machine :param target: Target DropNot Server IP/domain :param port: Port to access DropNot Server on """ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', filename='client.log', level=logging.INFO) self.sync_dir = sync_dir self.target = target self.port = port # Our key-value stores for client metadata self.file_db = SqliteDict('client_tracker.db', tablename='files', encode=json.dumps, decode=json.loads) self.folder_db = SqliteDict('client_tracker.db', tablename='folders', encode=json.dumps, decode=json.loads) Thread.__init__(self) def run(self): """ Start the DropNot Client :return: None """ # Start a DirectoryListener thread to scan for file changes dir_worker = DirectoryListener(dir=self.sync_dir, change_callback=self.on_change, file_db=self.file_db, folder_db=self.folder_db) Thread(name='dir_listener', target=DirectoryListener.scan_directory(dir_worker)) def on_change(self, change: ChangeType, path): """ Handle a change in the synced folder. Callback for the DirectoryListener :param change: Value of ChangeType defining what was changed :param path: Absolute path of changed file/folder :return: None """ # Switch windows path separators to forward slashes for rest URL rel_path = os.path.relpath(path, self.sync_dir).replace("\\", "/") # Handle change in synced folder if change == ChangeType.CreatedFolder: self.sync_new_folder(path, rel_path) elif change == ChangeType.DeletedFolder: self.sync_del_folder(path, rel_path) elif change == ChangeType.DeletedFile: self.sync_del_file(path, rel_path) elif change == ChangeType.CreatedFile: self.sync_new_file(path, rel_path) elif change == ChangeType.ModifiedFile: self.sync_edit_file(path, rel_path) else: logging.critical('Unsupported ChangeType') raise RuntimeError('Unsupported ChangeType') def sync_new_folder(self, path, rel_path): """ Synchronise creation of a folder to the server :param path: Path including sync directory on Client machine :param rel_path: Relative path from sync directory :return: None """ folder_encoding = FileUtils.get_folder_encoding(path, rel_path) resp = requests.post('http://{}:{}/sync/{}'.format( self.target, self.port, rel_path), json=repr(folder_encoding)) if resp.status_code == 200: logging.info('Folder synced to remote:{}'.format(rel_path)) folder_encoding.sync = True self.folder_db[path] = repr(folder_encoding) self.folder_db.commit() elif resp.status_code == 400 or resp.status_code == 400: logging.error('Folder sync unsuccessful{}'.format(rel_path)) self.folder_db[path] = repr(folder_encoding) self.folder_db.commit() else: logging.critical( 'Unsupported response status code from remote:{}'.format( resp.status_code)) raise NotImplementedError('Unsupported resp status code:', resp.status_code) def sync_del_folder(self, path, rel_path): """ Synchronise deletion of a folder to the server :param path: Path including sync directory on Client machine :param rel_path: Relative path from sync directory :return: None """ resp = requests.delete('http://{}:{}/sync/{}'.format( self.target, self.port, rel_path), json={"type": "folder"}) if resp.status_code == 200: logging.info('Folder removed from remote:{}'.format(rel_path)) self.folder_db.pop(path, None) self.folder_db.commit() elif resp.status_code == 400 or resp.status_code == 422: logging.error( 'Folder removal failed on remote:{}'.format(rel_path)) # todo: handle this? else: logging.critical( 'Unsupported response status code from remote:{}'.format( resp.status_code)) raise NotImplementedError('Unsupported resp status code:', resp.status_code) def sync_del_file(self, path, rel_path): """ Synchronise deletion of a file to the server :param path: Path including sync directory on Client machine :param rel_path: Relative path from sync directory :return: None """ resp = requests.delete('http://{}:{}/sync/{}'.format( self.target, self.port, rel_path), json={"type": "file"}) if resp.status_code == 200: logging.info('File removed from remote:{}'.format(rel_path)) self.file_db.pop(path, None) self.file_db.commit() elif resp.status_code == 400 or resp.status_code == 400: logging.error('File removal failed on remote:{}'.format(rel_path)) # todo: handle this? else: logging.critical( 'Unsupported response status code from remote:{}'.format( resp.status_code)) raise NotImplementedError('Unsupported resp status code:', resp.status_code) def sync_new_file(self, path, rel_path): """ Synchronise a new file to the server :param path: Path including sync directory on Client machine :param rel_path: Relative path from sync directory :return: None """ file_encoding = FileUtils.get_file_encoding(path, rel_path) file_metadata = FileUtils.get_metadata(file_encoding) if self.check_file_exists(file_metadata.md5): # Only send metadata, not the binary resp = requests.post('http://{}:{}/sync/{}'.format( self.target, self.port, rel_path), json=repr(file_metadata)) else: # Send metadata and binary resp = requests.post('http://{}:{}/sync/{}'.format( self.target, self.port, rel_path), json=repr(file_encoding)) if resp.status_code == 200: logging.info('File creation synced to remote:{}'.format(rel_path)) file_metadata.sync = True self.file_db[path] = repr(file_metadata) self.file_db.commit() elif resp.status_code == 400 or resp.status_code == 422: logging.error( 'File creation sync unsuccessful:{}'.format(rel_path)) self.file_db[path] = repr(file_metadata) self.file_db.commit() # todo: handle this? else: logging.critical( 'Unsupported response status code from remote:{}'.format( resp.status_code)) raise NotImplementedError('Unsupported resp status code:', resp.status_code) def sync_edit_file(self, path, rel_path): """ Synchronise a file edit to the server :param path: Path including sync directory on Client machine :param rel_path: Relative path from sync directory :return: None """ file_encoding = FileUtils.get_file_encoding(path, rel_path) file_metadata = FileUtils.get_metadata(file_encoding) if self.check_file_exists(file_metadata.md5): # Only send metadata, not the binary resp = requests.put('http://{}:{}/sync/{}'.format( self.target, self.port, rel_path), json=repr(file_metadata)) else: # Send metadata and binary resp = requests.put('http://{}:{}/sync/{}'.format( self.target, self.port, rel_path), json=repr(file_encoding)) if resp.status_code == 200: logging.info( 'File modification synced to remote:{}'.format(rel_path)) file_metadata.sync = True self.file_db[path] = repr(file_metadata) self.file_db.commit() elif resp.status_code == 400 or resp.status_code == 422: logging.info( 'File modification sync unsuccessful:{}'.format(rel_path)) self.file_db[path] = repr(file_metadata) self.file_db.commit() # todo: handle this? else: logging.critical( 'Unsupported response status code from remote:{}'.format( resp.status_code)) raise NotImplementedError('Unsupported resp status code:', resp.status_code) def check_file_exists(self, md5): """ Check whether a file matching the md5 already exists on the remote :param md5: md5 hash :return: True if an identical file exists, False otherwise """ resp = requests.get('http://{}:{}/sync/exists/{}'.format( self.target, self.port, md5)) if resp.status_code == 200: return True else: return False
class Gdrive: def __init__(self, config, token_path, cache_path): self.cfg = config self.token_path = token_path self.cache_path = cache_path self.token = None self.cache = None def first_run(self): # token file if not os.path.exists(self.token_path): # token.json does not exist, lets do the first run auth process print( "Visit %s and authorize against the account you wish to use" % self.authorize_url()) auth_code = raw_input('Enter authorization code: ') if self.first_access_token(auth_code) and self.token is not None: self.dump_token() else: logger.error( "Failed to authorize with the supplied client_id/client_secret/auth_code..." ) return False else: self.token = utils.load_json(self.token_path) # cache file self.cache = SqliteDict(self.cache_path, tablename='cache', encode=json.dumps, decode=json.loads, autocommit=False) return True def authorize_url(self): payload = { 'client_id': self.cfg['GDRIVE']['CLIENT_ID'], 'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob', 'response_type': 'code', 'access_type': 'offline', 'scope': 'https://www.googleapis.com/auth/drive' } url = 'https://accounts.google.com/o/oauth2/v2/auth?' + urlencode( payload) return url def first_access_token(self, auth_code): logger.info("Requesting access token for auth code %r", auth_code) payload = { 'code': auth_code, 'client_id': self.cfg['GDRIVE']['CLIENT_ID'], 'client_secret': self.cfg['GDRIVE']['CLIENT_SECRET'], 'grant_type': 'authorization_code', 'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob', } success, resp, data = self._make_request( 'https://www.googleapis.com/oauth2/v4/token', data=payload, headers={}, request_type='post') if success and resp.status_code == 200: logger.info("Retrieved first access token!") self.token = data self.token['page_token'] = '' return True else: logger.error("Error retrieving first access_token:\n%s", data) return False def refresh_access_token(self): logger.debug("Renewing access token...") payload = { 'refresh_token': self.token['refresh_token'], 'client_id': self.cfg['GDRIVE']['CLIENT_ID'], 'client_secret': self.cfg['GDRIVE']['CLIENT_SECRET'], 'grant_type': 'refresh_token', } success, resp, data = self._make_request( 'https://www.googleapis.com/oauth2/v4/token', data=payload, headers={}, request_type='post') if success and resp.status_code == 200 and 'access_token' in data: logger.info("Renewed access token!") refresh_token = self.token['refresh_token'] page_token = self.token['page_token'] self.token = data if 'refresh_token' not in self.token or not self.token[ 'refresh_token']: self.token['refresh_token'] = refresh_token self.token['page_token'] = page_token self.dump_token() return True else: logger.error("Error renewing access token:\n%s", data) return False def get_changes_first_page_token(self): success, resp, data = self._make_request( 'https://www.googleapis.com/drive/v3/changes/startPageToken', params={'supportsTeamDrives': self.cfg['GDRIVE']['TEAMDRIVE']}) if success and resp.status_code == 200: if 'startPageToken' not in data: logger.error( "Failed to retrieve startPageToken from returned startPageToken:\n%s", data) return False self.token['page_token'] = data['startPageToken'] self.dump_token() return True else: logger.error("Error retrieving first page token:\n%s", data) return False def get_changes(self): success, resp, data = self._make_request( 'https://www.googleapis.com/drive/v3/changes', params={ 'pageToken': self.token['page_token'], 'pageSize': 1000, 'includeRemoved': True, 'includeTeamDriveItems': self.cfg['GDRIVE']['TEAMDRIVE'], 'supportsTeamDrives': self.cfg['GDRIVE']['TEAMDRIVE'], 'fields': 'changes(file(md5Checksum,mimeType,modifiedTime,' 'name,parents,teamDriveId,trashed),' 'fileId,removed,teamDrive(id,name),' 'teamDriveId),newStartPageToken,nextPageToken' }) if success and resp.status_code == 200: # page token logic if data is not None and 'nextPageToken' in data: self.token['page_token'] = data['nextPageToken'] self.dump_token() elif data is not None and 'newStartPageToken' in data: self.token['page_token'] = data['newStartPageToken'] self.dump_token() else: logger.error( "Unexpected response while polling for changes from page %s:\n%s", str(self.token['page_token']), data) return False, data return True, data else: logger.error("Error getting page changes for page_token %r:\n%s", self.token['page_token'], data) return False, data def get_id_metadata(self, item_id, teamdrive_id=None): # return cache from metadata if available cached_metadata = self._get_cached_metdata(item_id) if cached_metadata: return True, cached_metadata # does item_id match teamdrive_id? if teamdrive_id is not None and item_id == teamdrive_id: success, resp, data = self._make_request( 'https://www.googleapis.com/drive/v3/teamdrives/%s' % str(item_id)) if success and resp.status_code == 200 and 'name' in data: # we successfully retrieved this teamdrive info, lets place a mimeType key in the result # so we know it needs to be cached data['mimeType'] = 'application/vnd.google-apps.folder' else: # retrieve file metadata success, resp, data = self._make_request( 'https://www.googleapis.com/drive/v3/files/%s' % str(item_id), params={ 'supportsTeamDrives': self.cfg['GDRIVE']['TEAMDRIVE'], 'fields': 'id,md5Checksum,mimeType,modifiedTime,name,parents,' 'trashed,teamDriveId' }) if success and resp.status_code == 200: return True, data else: logger.error("Error retrieving metadata for item %r:\n%s", item_id, data) return False, data def get_id_file_paths(self, item_id, teamdrive_id=None): file_paths = [] added_to_cache = 0 try: def get_item_paths(obj_id, path, paths, new_cache_entries, teamdrive_id=None): success, obj = self.get_id_metadata(obj_id, teamdrive_id) if not success: return new_cache_entries teamdrive_id = teamdrive_id if 'teamDriveId' not in obj else obj[ 'teamDriveId'] # add item object to cache if we know its not from cache if 'mimeType' in obj: # we know this is a new item fetched from the api, because the cache does not store this field self.add_item_to_cache( obj['id'], obj['name'], [] if 'parents' not in obj else obj['parents']) new_cache_entries += 1 if path.strip() == '': path = obj['name'] else: path = os.path.join(obj['name'], path) if 'parents' in obj and obj['parents']: for parent in obj['parents']: new_cache_entries += get_item_paths( parent, path, paths, new_cache_entries, teamdrive_id) if (not obj or 'parents' not in obj or not obj['parents']) and len(path): paths.append(path) return new_cache_entries return new_cache_entries added_to_cache += get_item_paths(item_id, '', file_paths, added_to_cache, teamdrive_id) if added_to_cache: logger.debug("Dumping cache due to new entries!") self.dump_cache() if len(file_paths): return True, file_paths else: return False, file_paths except Exception: logger.exception("Exception retrieving filepaths for '%s': ", item_id) return False, [] # cache def add_item_to_cache(self, item_id, item_name, item_parents): if item_id not in self.cache: logger.info("Added '%s' to cache: %s", item_id, item_name) self.cache[item_id] = {'name': item_name, 'parents': item_parents} return def remove_item_from_cache(self, item_id): if self.cache.pop(item_id, None): return True return False # dump jsons def dump_token(self): utils.dump_json(self.token_path, self.token) return def dump_cache(self): self.cache.commit() return ############################################################ # INTERNALS ############################################################ # cache def _get_cached_metdata(self, item_id): if item_id in self.cache: return self.cache[item_id] return None # requests @backoff.on_predicate(backoff.expo, lambda x: not x[0] and ('error' in x[2] and 'code' in x[2]['error'] and x[2] ['error']['code'] != 401), max_tries=8) def _make_request(self, url, headers=None, data=None, params=None, request_type='get'): refreshed_token = False while True: if headers is None and self.token: auth_headers = { 'Authorization': 'Bearer %s' % self.token['access_token'], } else: auth_headers = {} resp = None if request_type == 'get': resp = requests.get( url, params=params, headers=headers if headers is not None else auth_headers, timeout=30) elif request_type == 'post': resp = requests.post( url, data=data, headers=headers if headers is not None else auth_headers, timeout=30) else: return False, resp, { 'error': { 'code': 401, 'message': 'Invalid request_type was supplied to _make_request' } } # response logic try: data = resp.json() except ValueError: logger.exception( "Exception while decoding response from Google Drive for data:\n%s\nTraceback: ", resp.text) return False, resp, { 'error': { 'code': resp.status_code, 'message': 'Failed to json decode Google Drive response' } } if 'error' in data and 'code' in data['error'] and ( 'message' in data['error'] and 'Invalid Credentials' in data['error']['message']): # the token has expired. if not refreshed_token: refreshed_token = True self.refresh_access_token() continue else: # attempt was already made to refresh token return False, resp, data if resp.status_code == 200: return True, resp, data else: return False, resp, data
class BucketObject(object): def __init__(self, object_id=None, metadata={}, bucket=None): self.object_id = object_id self._bucket = bucket if bucket and isinstance(bucket, Bucket) and bucket.exists(): self._bucket = bucket self._temp_path = os.path.join(self._bucket.bucket_path, 'tmp') self._data_path = os.path.join(self._bucket.bucket_path, 'data') else: raise falcon.HTTPBadRequest( title='InvalidBucketName', description= 'The specified bucket is not valid' ) # metadata key used for object identification in the storage self._metadata = dict() if 'object-key' in self._bucket.metadata and self._bucket.metadata['object-key'] in ['content-md5', 'content-sha1']: self.OBJECT_KEY_BASE=self._bucket.metadata['object-key'] else: self.OBJECT_KEY_BASE='content-sha1' self._objects_metadata = SqliteDict(os.path.join(bucket.bucket_path,'metadata.sqlite'), 'objects', autocommit=True) @property def metadata(self): if self.object_id and self.object_id in self._objects_metadata: return dict(self._objects_metadata[self.object_id]) else: return self._metadata @metadata.setter def metadata(self, value): if value and isinstance(value, dict): # object_id doesn't set but object_id is in the value if not self.object_id and self.OBJECT_KEY_BASE in value: self.object_id = value[self.OBJECT_KEY_BASE] # object_id is assigned if self.object_id: self._objects_metadata[self.object_id] = value # no object_id, stored temporary else: self._metadata = value else: raise RuntimeError('Incorrect metadata type. Found "%s", expected "dict"') % type(value) @property def filepath(self): ''' returns the path to file in storage ''' if self.object_id: return os.path.join(self._data_path, pairtree_path(self.object_id)) else: return os.path.join(self._data_path, pairtree_path(self._metadata[self.OBJECT_KEY_BASE])) def exists(self): ''' returns True if the object is in the storage ''' if self.object_id and self.object_id in self._objects_metadata: return True else: return False def store(self, stream=None): ''' store object into bucket ''' # TODO: for later analysis # if not stream or not isinstance(stream, gunicorn.http.body.Body): # raise falcon.HTTPInternalServerError( # title = 'InvalidStreamType', # description = 'The Stream type is invalid, %s' % type(stream) # ) if not stream: raise falcon.HTTPInternalServerError( title = 'FileEmpty', description = 'Attempt to store empty file' ) temp_filepath = None with tempfile.NamedTemporaryFile(dir=self._temp_path, mode="wb", delete=False) as _file: temp_filepath = _file.name while True: chunk = stream.read(4096) if not chunk: break _file.write(chunk) hashes = { 'content-md5': self.filehash(temp_filepath, hashlib.md5()), 'content-sha1': self.filehash(temp_filepath, hashlib.sha1()), 'content-sha256': self.filehash(temp_filepath, hashlib.sha256()) } if self.validate(temp_filepath, hashes): self._metadata.update(hashes) self.object_id = self._metadata[self.OBJECT_KEY_BASE] if hashes[self.OBJECT_KEY_BASE] not in self._objects_metadata: movefile(temp_filepath, self.filepath) self.metadata = self._metadata else: os.remove(temp_filepath) raise falcon.HTTPConflict( title="ObjectAlreadyExists", description="The object already exists into the storage.", headers={ 'content-md5': self.metadata['content-md5'], 'content-sha1': self.metadata['content-sha1'], 'content-sha256': self.metadata['content-sha256'], } ) def filehash(self, filepath, hashfunc): ''' returns file hash ''' block_size = 2 ** 20 with open(filepath, 'rb') as _file: while True: data = _file.read(block_size) if not data: break hashfunc.update(data) return hashfunc.hexdigest() def validate(self, filepath, filehashes): ''' validate recieved file object ''' if os.path.getsize(filepath) == 0: os.remove(filepath) raise falcon.HTTPBadRequest( title='ZeroContentLength', description='The content size is 0' ) if 'content-length' in self._metadata and \ os.path.getsize(filepath) != self._metadata['content-length']: os.remove(filepath) raise falcon.HTTPBadRequest( title='BadContentLength', description='The Content-length did not match' ) if 'content-md5' in self._metadata and \ filehashes['md5'] != self._metadata['content-md5']: os.remove(filepath) raise falcon.HTTPBadRequest( title='BadDigest', description='The Content-MD5 did not match' ) if 'content-sha1' in self._metadata and \ filehashes['sha1'] != self._metadata['content-sha1']: os.remove(filepath) raise falcon.HTTPBadRequest( title='BadDigest', description='The Content-SHA1 did not match' ) return True def delete(self): ''' delete object ''' if self.exists(): if os.path.exists(self.filepath): os.remove(self.filepath) else: raise falcon.HTTPNotFound() self._objects_metadata.pop(self.object_id) def info(self): ''' returns object's metadata ''' if self.object_id and self.object_id in self._objects_metadata: return self._objects_metadata[self.object_id] else: return self._metadata
class MyForm(QMainWindow): def __init__(self): super().__init__() self.ui = Ui_MainWindow() self.ui.setupUi(self) self.ui.button_load_data.clicked.connect(self.load) self.ui.edit_database_path.editingFinished.connect(self.load_from_lineEdit) self.ui.button_save.clicked.connect(self.save) self.ui.button_add.clicked.connect(self.add_entry) self.ui.button_plan.clicked.connect(self.gen_week_table) def closeEvent(self,event): close = QMessageBox() close.setText("Speichern ?") close.setStandardButtons(QMessageBox.Yes | QMessageBox.Cancel) close = close.exec() if close == QMessageBox.Yes: self.save() event.accept() else: event.accept() def load(self): # Load SQLdict #try: self.fname = QFileDialog.getOpenFileName(self, 'Open file','','Database *.sqlite') # gt file path self.ui.edit_database_path.setText('Loaded database: ' + self.fname[0]) # set path to LineEdit self.Kochbuch = SqliteDict(self.fname[0], autocommit=True) # Lade Haupt dictionary/ Gerichte self.create_content_table() #except: #print('Cannot load specified file!\nError in main.load()') #pass def load_from_lineEdit(self): # Load from LineEdit try: self.fname[0] = self.ui.edit_database_path.text()[17:] # get text without prefix self.Kochbuch = SqliteDict(self.fname[0], autocommit=True) # Lade Haupt dictionary/ Gerichte self.create_content_table() except: print('Cannot load specified file!\nError in main.load_from_lineEdit()') pass def save(self): # get items from content_table, update Kochbuch and commit it to database self.Kochbuch.clear() table = self.ui.content_table header_items = [table.model().headerData(i, Qt.Horizontal) for i in range(table.model().columnCount())] for row_index in range(self.ui.content_table.rowCount()): # Every row is one dish/gericht temp_dict = dict() for col_index, item in enumerate(header_items): temp_dict[item] = table.cellWidget(row_index,col_index).text() self.add_gericht(temp_dict) def add_entry(self): # Add empty entry to table row_cnt = self.ui.content_table.rowCount() col_cnt = self.ui.content_table.columnCount() self.ui.content_table.insertRow(row_cnt) for col_index in range(col_cnt): self.ui.content_table.setCellWidget(row_cnt, col_index, QLineEdit()) if col_index == col_cnt - 1: # Delete Option self.ui.content_table.setCellWidget(row_cnt, col_index, QPushButton('Delete')) self.ui.content_table.cellWidget(row_cnt, col_index).clicked.connect(self.remove_entry) def remove_entry(self): table = self.ui.content_table # --------------Remove Row------------ column = table.currentColumn() row = table.currentRow() table.removeRow(row) # -------------Remove dict entry-------- #name = table.cellWidget(row,0).text() #self.del_gericht(name) def create_content_table(self): # Creates the Widgets inside the Table table = self.ui.content_table table.setRowCount(len(self.Kochbuch)) header_items = [table.model().headerData(i, Qt.Horizontal) for i in range(table.model().columnCount())] row_label = [] col_cnt = table.model().columnCount() for row_index, val in enumerate(self.Kochbuch.items()): #row_label.append(str(row_index + 1) + ' ' + str(val[0])) for col_index in range(col_cnt): table.setCellWidget(row_index,col_index,QLineEdit()) if col_index == col_cnt - 1: # Add Delete Button table.setCellWidget(row_index, col_index, QPushButton('Delete')) table.cellWidget(row_index,col_index).clicked.connect(self.remove_entry) #self.ui.content_table.setVerticalHeaderLabels(row_label) self.set_text_to_table(header_items) def set_text_to_table(self,header_items): table = self.ui.content_table for row_index, val in enumerate(self.Kochbuch.items()): table.cellWidget(row_index, 0).setText(val[0]) # Name column/ set Name #print(val[1].values()) for col_index, item in enumerate(header_items[1:]): try: table.cellWidget(row_index, col_index + 1).setText(val[1][item]) except KeyError: if item == None: # Used, that the delete button text will not be overwritten pass else: # Set unfilled category empty table.cellWidget(row_index, col_index + 1).setText('') def add_gericht(self, entries:dict): # Olf func args: name: str, Fisch: bool, Nudeln: bool, Vortag: bool, SE: bool, WE: bool, WE_Wichtung: float, Wichtung: float # Gerichte werden gespeichert in dict() # Jedes Gericht wird dabei Kategorisiert in: # - Fisch: Bool # - Nudeln: Bool # - Vortag: Bool (Wenn Sonntags viel Bestellt wird das Essen vom Vortag machen) # - SE: Bool (Sonntagsessen) # - WE: Bool (Wochenendessen, wie z.B Holen/Bestellen) # - WE_Wichtung: Float (Jedes Gericht soll die chance haben am WE dran zu kommen, Holen/Bestellen oder z.B. Rolladen sollen bevorzugt werden) # - Wichtung: Float (Warkeit des Gerichtes ausgewählt zu werden, um doppelte zu vermeiden) 1.0 = Kommt dran; 0 = wird nicht dran kommen # etc. # ------------------------------------------------------------------------------- # Tortillas = dict() # Ein Gericht dict() # Tortillas['Fisch'] = False # Tortillas['Nudeln'] = False # Tortillas['Vortag'] = False # Tortillas['SE'] = False # Tortillas['WE'] = False # Tortillas['WE_Wichtung'] = 0.1 # Tortillas['Wichtung'] = 1.0 # Gerichte['Tortillas'] = Tortillas # ------------------------------------------------------------------------------- '''name_dict = dict() name_dict['Fisch'] = Fisch name_dict['Nudeln'] = Nudeln name_dict['Vortag'] = Vortag name_dict['SE'] = SE name_dict['WE'] = WE name_dict['WE_Wichtung'] = WE_Wichtung name_dict['Wichtung'] = Wichtung''' name = entries['Name'] self.Kochbuch[name] = entries def del_gericht(self,name): self.Kochbuch.pop(name) def update_kochbuch(self, Kochbuch: dict, name: str, kategorie: str, value: any): # Da hier mit dict im dict gearbeitet wird und dasa äußerste dict ein SQLdict ist, # müssen hier die Einträge ein bisschen umständlich verändert werden, um vom RAM in die SQL datei zu schreiben update = Kochbuch[name] update[kategorie] = value Kochbuch[name] = update def choose(self,dishes): choosed_dish = np.asarray(dishes) Wichtungen = choosed_dish[:,1].astype(np.float) #choosed_dish = np.sort(choosed_dish) # Find maximas in Wichtung column max_indizes = np.where(Wichtungen==np.amax(Wichtungen)) finds = [] for i in max_indizes[0]: finds.append(choosed_dish[i]) # Choose dish # If len of finds > 1 use random int to choose dish if len(finds) > 1: dish_index = random.randint(0,len(finds) - 1) return finds[dish_index] else: return finds #print('test','\n',choosed_dish) def gen_week_table(self): # generate table of dishes day wise dishes = [i for i in self.Kochbuch.items()] dishes_cnt = len(dishes) usable_dishes = [] possible_dishes_mon = [] possible_dishes_tue = [] possible_dishes_wed = [] possible_dishes_thu = [] possible_dishes_fri = [] possible_dishes_sat = [] possible_dishes_sun = [] saison = 'Winter' for index, dish in enumerate(dishes): dish = dish[1] # Perform standard check, to reduce dishes according to seasons and weigth #if float(dish['Wichtung']) > 0.7 and dish['Saison'] == saison: # Standard check if float(dish['Wichtung']) > 0.7 and (dish['Saison'] == saison or dish['Saison'] == 'None'): usable_dishes.append(dish) # -----------Monday------------- # ------------------------------ # Mondays should prefer Nudeln ---> Boni for Nudeln == True if dish['Fisch'] == 'False': if dish['Nudeln'] == 'True': possible_dishes_mon.append([dish['Name'], float(dish['Wichtung']) + 0.3]) else: possible_dishes_mon.append([dish['Name'], float(dish['Wichtung'])]) #----------------------------------------------------------------------------- # -----------Tuesday/Wednesday/Thursday------------- # -------------------------------------------------- # Days without preferations if dish['Fisch'] == 'False' and dish['Vortag'] == 'False': # Standard check possible_dishes_tue.append([dish['Name'], float(dish['Wichtung'])]) possible_dishes_wed.append([dish['Name'], float(dish['Wichtung'])]) possible_dishes_thu.append([dish['Name'], float(dish['Wichtung'])]) # ----------------------------------------------------------------------------- # -----------Friday------------- # ------------------------------ # Fish prefered if dish['WE'] == 'True' and dish['SE'] == 'False': if dish['Fisch'] == 'True': possible_dishes_fri.append([dish['Name'], float(dish['Wichtung']) + 0.3]) else: possible_dishes_fri.append([dish['Name'], float(dish['Wichtung'])]) # ----------------------------------------------------------------------------- # -----------Saturday------------- # -------------------------------- # WE category prefered if dish['Fisch'] == 'False' and dish['SE'] == 'False' and dish['Vortag'] == 'False': if dish['WE'] == 'True': possible_dishes_sat.append([dish['Name'], float(dish['Wichtung']) + 0.3]) else: possible_dishes_sat.append([dish['Name'], float(dish['Wichtung'])]) # ----------------------------------------------------------------------------- # -----------Sunday------------- # ------------------------------ # SE highly prefered if dish['Fisch'] == 'False' and dish['Vortag'] == 'False': if dish['SE'] == 'True': possible_dishes_sun.append([dish['Name'], float(dish['Wichtung']) + 0.5]) else: possible_dishes_sun.append([dish['Name'], float(dish['Wichtung'])]) print('============================================================================') print('=================================Wochenplan=================================') print('============================================================================') print('Monday: ', self.choose(possible_dishes_mon)[0]) print('----------------------------------------------------------------------------') print('Tuesday: ', self.choose(possible_dishes_tue)[0]) print('----------------------------------------------------------------------------') print('Wednesday: ', self.choose(possible_dishes_wed)[0]) print('----------------------------------------------------------------------------') print('Thurday: ', self.choose(possible_dishes_thu)[0]) print('----------------------------------------------------------------------------') print('Friday: ', self.choose(possible_dishes_fri)[0]) print('----------------------------------------------------------------------------') print('Saturday: ', self.choose(possible_dishes_sat)[0]) print('----------------------------------------------------------------------------') print('Sunday: ', self.choose(possible_dishes_sun)[0]) print('============================================================================') #print(self.choose(possible_dishes_mon)[0]) Speiseplan = collections.OrderedDict()
class SessionState: def __init__(self, cache_file: Path, cache_key: str, redis: str, user_requested=False): self.user_requested = user_requested self._cache_file = cache_file self._cache_key = cache_key self._cache: Optional[SqliteDict] = None random.seed() self._session_key = random.randint(0, 999999) self._redis = Redis(host=redis) if not user_requested: self._open() if self._cache_key != self._cache.get("_cache_key_", None): self._cache.close() self._cache: Optional[SqliteDict] = None self._cache_file.unlink() self._open() self._cache["_cache_key_"] = self._cache_key self.session = Session() # noinspection PyTypeChecker self.session.mount( 'https://', HTTPAdapter(max_retries=Retry(total=3, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]))) self.sites = {} self.wikidata = Sparql() self.primary_site = self.get_site(primary_domain) def __enter__(self): return self def __exit__(self, typ, value, traceback): self.session.close() if self._cache is not None: self._cache.close() self._cache = None print(f'Closed SQL connection for {self._session_key} at {datetime.utcnow()}') def _open(self): if self._cache is None: print(f'Opening SQL connection for {self._session_key} at {datetime.utcnow()}') self._cache_file.parent.mkdir(parents=True, exist_ok=True) self._cache = SqliteDict(self._cache_file, autocommit=True) def get_site(self, domain: Domain) -> WikiSite: try: return self.sites[domain] except KeyError: # noinspection PyTypeChecker site = WikiSite(domain, self.session, domain == primary_domain) if self.user_requested: site.maxlag = None self.sites[domain] = site return site def delete_cached_items(self, prefix: str) -> None: self._open() for vv in {v for v in self._cache.keys() if v.startswith(prefix)}: del self._cache[vv] def del_obj(self, key: str) -> Any: self._redis.delete(self.redis_key(key)) self._open() print(f"%% del {key}") return self._cache.pop(key, None) def load_obj(self, key: str, default: Any = None) -> Any: value = self._redis.get(self.redis_key(key)) if value is not None: return loads(value) self._open() print(f"%% load {key}") value = self._cache.get(key, default) self._redis.set(self.redis_key(key), dumps(value)) return value def save_obj(self, key: str, value: Any): self._open() print(f"%% save {key}") self._cache[key] = value self._redis.set(self.redis_key(key), dumps(value)) def redis_key(self, key: str): return self._cache_key + key