def get_server_log_entries(self, environ, session_token): try: session = self.sessions[session_token] self.ui.set_progress_text("Sending log entries...") log_entries = session.database.log_entries_to_sync_for(\ session.client_info["machine_id"], session.client_info["interested_in_old_reps"]) number_of_entries = session.database.\ number_of_log_entries_to_sync_for(\ session.client_info["machine_id"], session.client_info["interested_in_old_reps"]) for buffer in self._stream_log_entries(log_entries, number_of_entries): yield buffer except: yield self.handle_error(session, traceback_string()) # Now that all the data is underway to the client, we can already # start applying the client log entries. If there are errors that # occur, we save them and communicate them to the client in # 'get_sync_finish'. try: self.ui.set_progress_text("Applying log entries...") # First, dump to the science log, so that we can skip over the new # logs in case the client uploads them. session.database.dump_to_science_log() for log_entry in session.client_log: session.database.apply_log_entry(log_entry) # Skip over the logs that the client promised to upload. if session.client_info["upload_science_logs"]: session.database.skip_science_log() except: session.apply_error = traceback_string()
def content(): try: for buffer in self.stream_binary_file(filename): yield buffer binary_format.clean_up() except: yield self.handle_error(session, traceback_string())
def _deploy_without_asking(): if env.offline: commands = OFFLINE_DEPLOY_COMMANDS else: commands = ONLINE_DEPLOY_COMMANDS try: for index, command in enumerate(commands): deploy_checkpoint(index, command.func_name, execute_with_timing, command) except PreindexNotFinished: mail_admins( " You can't deploy to {} yet. There's a preindex in process.". format(env.environment), ("Preindexing is taking a while, so hold tight " "and wait for an email saying it's done. " "Thank you for using AWESOME DEPLOY.")) except Exception: execute_with_timing( mail_admins, "Deploy to {environment} failed. Try resuming with " "fab {environment} deploy:resume=yes.".format( environment=env.environment), traceback_string()) # hopefully bring the server back to life silent_services_restart() raise else: execute_with_timing(release.update_current) silent_services_restart() execute_with_timing(release.record_successful_release) execute_with_timing(release.record_successful_deploy) clear_cached_deploy()
def get_server_entire_database_binary(self, environ, session_token): try: session = self.sessions[session_token] self.ui.set_progress_text("Sending entire binary database...") binary_format = self.binary_format_for(session) filename = binary_format.binary_filename(\ session.client_info["store_pregenerated_data"], session.client_info["interested_in_old_reps"]) global mnemosyne_content_length mnemosyne_content_length = os.path.getsize(filename) # Since we want to modify the headers in this function, we cannot # use 'yield' directly to stream content, but have to add one layer # of indirection: http://www.cherrypy.org/wiki/ReturnVsYield # # Since we return an iterator, we also need to re-encapsulate our # code in a try block. def content(): try: for buffer in self.stream_binary_file(filename): yield buffer binary_format.clean_up() except: yield self.handle_error(session, traceback_string()) return content() # This is a full sync, we don't need to apply client log # entries here. except: return self.handle_error(session, traceback_string())
def get_server_binary_file(self, environ, session_token, filename): try: session = self.sessions[session_token] global mnemosyne_content_length socket = environ["wsgi.input"] filename = unicode(filename, "utf-8") # Make sure a malicious client cannot access anything outside # of the media directory. filename = filename.replace("../", "").replace("..\\", "") filename = filename.replace("/..", "").replace("\\..", "") filename = os.path.join(session.database.data_dir(), filename) file_size = os.path.getsize(filename) mnemosyne_content_length = file_size # Since we want to modify the headers in this function, we cannot # use 'yield' directly to stream content, but have to add one layer # of indirection: http://www.cherrypy.org/wiki/ReturnVsYield # # We don't have progress bars here, as 'get_server_media_file' # gets called too frequently, and this would slow down the UI. # # Since we return an iterator, we also need to re-encapsulate our # code in a try block. def content(): try: for buffer in self.stream_binary_file(\ filename, progress_bar=False): yield buffer except: yield self.handle_error(session, traceback_string()) return content() except: return self.handle_error(session, traceback_string())
def content(): try: for buffer in self.stream_binary_file(\ filename, progress_bar=False): yield buffer except: yield self.handle_error(session, traceback_string())
def get_server_media_filenames(self, environ, session_token, redownload_all=False): try: session = self.sessions[session_token] global mnemosyne_content_length mnemosyne_content_length = 0 self.ui.set_progress_text("Sending media files...") # Send list of filenames in the format <mediadir>/<filename>, i.e. # relative to the data_dir. Note we always use / internally. subdir = os.path.basename(session.database.media_dir()) if redownload_all in ["1", "True", "true"]: filenames = [subdir + "/" + filename for filename in \ session.database.all_media_filenames()] else: filenames = [subdir + "/" + filename for filename in \ session.database.media_filenames_to_sync_for(\ session.client_info["machine_id"])] if len(filenames) == 0: return "" for filename in filenames: mnemosyne_content_length += os.path.getsize((os.path.join(\ session.database.data_dir(), filename))) return "\n".join(filenames).encode("utf-8") except: return self.handle_error(session, traceback_string())
def get_server_generate_log_entries_for_settings(\ self, environ, session_token): try: session = self.sessions[session_token] session.database.generate_log_entries_for_settings() return self.text_format.repr_message("OK") except: return self.handle_error(session, traceback_string())
def get_sync_cancel(self, environ, session_token): try: self.ui.set_progress_text("Sync cancelled!") self.cancel_session_with_token(session_token) return self.text_format.repr_message("OK") except: session = self.sessions[session_token] return self.handle_error(session, traceback_string())
def get_sync_finish(self, environ, session_token): try: session = self.sessions[session_token] if session.apply_error is not None: return self.handle_error(session, session.apply_error) self.ui.set_progress_text("Sync finished!") self.close_session_with_token(session_token) # Now is a good time to garbage-collect dangling sessions. self.expire_old_sessions() return self.text_format.repr_message("OK") except: return self.handle_error(session, traceback_string())
def get_server_entire_database(self, environ, session_token): try: session = self.sessions[session_token] self.ui.set_progress_text("Sending entire database...") session.database.dump_to_science_log() log_entries = session.database.all_log_entries(\ session.client_info["interested_in_old_reps"]) number_of_entries = session.database.number_of_log_entries(\ session.client_info["interested_in_old_reps"]) for buffer in self._stream_log_entries(log_entries, number_of_entries): yield buffer except: yield self.handle_error(session, traceback_string())
def get_server_check_media_files(self, environ, session_token): # We check if files were updated outside of the program, or if # media files need to be generated dynamically, e.g. latex. This # can generate MEDIA_EDITED log entries, so it should be done first. try: session = self.sessions[session_token] if self.check_for_edited_local_media_files: self.ui.set_progress_text("Checking for edited media files...") session.database.check_for_edited_media_files() # Always create media files, otherwise they are not synced across. self.ui.set_progress_text("Dynamically creating media files...") session.database.dynamically_create_media_files() return self.text_format.repr_message("OK") except: return self.handle_error(session, traceback_string())
def put_client_binary_file(self, environ, session_token, filename): try: session = self.sessions[session_token] socket = environ["wsgi.input"] size = int(environ["CONTENT_LENGTH"]) filename = unicode(filename, "utf-8") # Make sure a malicious client cannot overwrite anything outside # of the media directory. filename = filename.replace("../", "").replace("..\\", "") filename = filename.replace("/..", "").replace("\\..", "") filename = os.path.join(session.database.data_dir(), filename) # We don't have progress bars here, as 'put_client_binary_file' # gets called too frequently, and this would slow down the UI. self.download_binary_file(environ["wsgi.input"], filename, size, progress_bar=False) return self.text_format.repr_message("OK") except: return self.handle_error(session, traceback_string())
def put_client_entire_database_binary(self, environ, session_token): try: session = self.sessions[session_token] self.ui.set_progress_text("Getting entire binary database...") filename = session.database.path() session.database.abandon() file_size = int(environ["CONTENT_LENGTH"]) self.download_binary_file(\ environ["wsgi.input"], filename, file_size) session.database.load(filename) session.database.change_user_id(session.client_info["user_id"]) session.database.create_if_needed_partnership_with(\ session.client_info["machine_id"]) session.database.remove_partnership_with(self.machine_id) # Next sync with a third party should be a full sync too. session.database.reset_partnerships() return self.text_format.repr_message("OK") except: return self.handle_error(session, traceback_string())
def get_server_media_filenames(self, environ, session_token, redownload_all=False): try: session = self.sessions[session_token] global mnemosyne_content_length mnemosyne_content_length = 0 self.ui.set_progress_text("Sending media files...") if redownload_all in ["1", "True", "true"]: filenames = list(session.database.all_media_filenames()) else: filenames = list(session.database.media_filenames_to_sync_for(\ session.client_info["machine_id"])) if len(filenames) == 0: return "" for filename in filenames: mnemosyne_content_length += os.path.getsize(\ os.path.join(session.database.media_dir(), filename)) return "\n".join(filenames).encode("utf-8") except: return self.handle_error(session, traceback_string())
def put_client_log_entries(self, environ, session_token): try: session = self.sessions[session_token] self.ui.set_progress_text("Receiving log entries...") socket = environ["wsgi.input"] element_loop = self.text_format.parse_log_entries(socket) session.number_of_client_entries = int(element_loop.next()) if session.number_of_client_entries == 0: return self.text_format.repr_message("OK") self.ui.set_progress_range(session.number_of_client_entries) self.ui.set_progress_update_interval(\ session.number_of_client_entries/50) for log_entry in element_loop: session.client_log.append(log_entry) if log_entry["type"] not in self.dont_cause_conflict: if "fname" in log_entry: log_entry["o_id"] = log_entry["fname"] session.client_o_ids.append(log_entry["o_id"]) self.ui.set_progress_value(len(session.client_log)) # If we haven't downloaded all entries yet, tell the client # it's OK to continue. if len(session.client_log) < session.number_of_client_entries: return self.text_format.repr_message("Continue") # Now we have all the data from the client and we can determine # whether there are conflicts. for log_entry in session.database.log_entries_to_sync_for(\ session.client_info["machine_id"]): if not log_entry: continue # Irrelevent entry for card-based clients. if "fname" in log_entry: log_entry["o_id"] = log_entry["fname"] if log_entry["type"] not in self.dont_cause_conflict and \ log_entry["o_id"] in session.client_o_ids: return self.text_format.repr_message("Conflict") if session.database.is_empty(): session.database.change_user_id(session.client_info["user_id"]) return self.text_format.repr_message("OK") except: return self.handle_error(session, traceback_string())
def get_server_archive_filenames(self, environ, session_token): try: session = self.sessions[session_token] global mnemosyne_content_length mnemosyne_content_length = 0 self.ui.set_progress_text("Sending archive files...") # Send list of filenames in the format "archive"/<filename>, i.e. # relative to the data_dir. Note we always use / internally. archive_dir = os.path.join(session.database.data_dir(), "archive") if not os.path.exists(archive_dir): return "" filenames = ["archive/" + filename for filename in \ os.listdir(archive_dir) if os.path.isfile\ (os.path.join(archive_dir, filename))] if len(filenames) == 0: return "" for filename in filenames: mnemosyne_content_length += os.path.getsize(os.path.join(\ session.database.data_dir(), filename)) return "\n".join(filenames).encode("utf-8") except: return self.handle_error(session, traceback_string())
def hotfix_deploy(): """ deploy ONLY the code with no extra cleanup or syncing for small python-only hotfixes """ if not console.confirm('Are you sure you want to deploy to {env.environment}?'.format(env=env), default=False) or \ not console.confirm('Did you run "fab {env.environment} preindex_views"? '.format(env=env), default=False) or \ not console.confirm('HEY!!!! YOU ARE ONLY DEPLOYING CODE. THIS IS NOT A NORMAL DEPLOY. COOL???', default=False): utils.abort('Deployment aborted.') _require_target() run('echo ping!') # workaround for delayed console response try: execute(release.update_code, env.deploy_metadata.deploy_ref, True) except Exception: execute(mail_admins, "Deploy failed", traceback_string()) # hopefully bring the server back to life silent_services_restart(use_current_release=True) raise else: silent_services_restart(use_current_release=True) execute(release.record_successful_deploy)
def sync(self, server, port, username, password): try: self.server = socket.gethostbyname(server) self.port = port if self.do_backup: self.ui.set_progress_text("Creating backup...") backup_file = self.database.backup() # We check if files were edited outside of the program. This can # generate EDITED_MEDIA_FILES log entries, so it should be done # first. if self.check_for_edited_local_media_files: self.ui.set_progress_text("Checking for edited media files...") self.database.check_for_edited_media_files() # Always create media files, otherwise they are not synced across. self.ui.set_progress_text("Dynamically creating media files...") self.database.dynamically_create_media_files() # Set timeout long enough for e.g. a slow NAS waking from # hibernation. socket.setdefaulttimeout(60) self.login(username, password) # Generating media files at the server side could take some time, # so we update the timeout. self.con = None socket.setdefaulttimeout(15 * 60) self.get_server_check_media_files() # Do a full sync after either the client or the server has restored # from a backup. if self.database.is_sync_reset_needed(\ self.server_info["machine_id"]) or \ self.server_info["sync_reset_needed"] == True: self.resolve_conflicts(restored_from_backup=True) # First sync, fetch database from server. elif self.database.is_empty(): self.get_server_media_files() if self.server_info["supports_binary_transfer"]: self.get_server_entire_database_binary() else: self.get_server_entire_database() self.get_sync_finish() # Fetch config settings. self.login(username, password) self.get_server_generate_log_entries_for_settings() self.get_server_log_entries() self.get_sync_finish() # First sync, put binary database to server if supported. elif not self.database.is_empty() and \ self.server_info["is_database_empty"] and \ self.supports_binary_upload(): self.put_client_media_files(reupload_all=True) self.put_client_entire_database_binary() self.get_sync_finish() # Upload config settings. self.login(username, password) self.database.generate_log_entries_for_settings() self.put_client_log_entries() self.get_server_log_entries() self.get_sync_finish() else: # Upload local changes and check for conflicts. result = self.put_client_log_entries() if result == "OK": self.put_client_media_files() self.get_server_media_files() self.get_server_log_entries() self.get_sync_finish() else: self.resolve_conflicts() self.ui.show_information("Sync finished!") except Exception, exception: self.ui.close_progress() serious = True if type(exception) == type(socket.gaierror()): self.ui.show_error("Could not find server!") serious = False elif type(exception) == type(socket.error()): self.ui.show_error("Could not connect to server!") serious = False elif type(exception) == type(socket.timeout()): self.ui.show_error("Timeout while waiting for server!") elif type(exception) == type(SyncError()): self.ui.show_error(str(exception)) serious = False elif type(exception) == type(SeriousSyncError()): self.ui.show_error(str(exception)) else: self.ui.show_error(traceback_string()) if serious and self.do_backup: # Only serious errors should result in the need for a full # sync next time. self.ui.show_error("Sync failed, restoring from backup. " + \ "The next sync will need to be a full sync.") if backup_file: self.database.restore(backup_file)
def sync(self, server, port, username, password): try: self.server = socket.gethostbyname(server) self.port = port if self.do_backup: self.ui.set_progress_text("Creating backup...") backup_file = self.database.backup() # We check if files were edited outside of the program. This can # generate EDITED_MEDIA_FILES log entries, so it should be done # first. if self.check_for_edited_local_media_files: self.ui.set_progress_text("Checking for edited media files...") self.database.check_for_edited_media_files() # Always create media files, otherwise they are not synced across. self.ui.set_progress_text("Dynamically creating media files...") self.database.dynamically_create_media_files() # Set timeout long enough for e.g. a slow NAS waking from # hibernation. socket.setdefaulttimeout(60) self.login(username, password) # Generating media files at the server side could take some time, # so we update the timeout. self.con = None socket.setdefaulttimeout(15*60) self.get_server_check_media_files() # Do a full sync after either the client or the server has restored # from a backup. if self.database.is_sync_reset_needed(\ self.server_info["machine_id"]) or \ self.server_info["sync_reset_needed"] == True: self.resolve_conflicts(restored_from_backup=True) # First sync, fetch database from server. elif self.database.is_empty(): self.get_server_media_files(redownload_all=True) self.get_server_archive_files() if self.server_info["supports_binary_transfer"]: self.get_server_entire_database_binary() else: self.get_server_entire_database() self.get_sync_finish() # Fetch config settings. self.login(username, password) self.get_server_generate_log_entries_for_settings() self.get_server_log_entries() self.get_sync_finish() # First sync, put binary database to server if supported. elif not self.database.is_empty() and \ self.server_info["is_database_empty"] and \ self.supports_binary_upload(): self.put_client_media_files(reupload_all=True) self.put_client_archive_files() self.put_client_entire_database_binary() self.get_sync_finish() # Upload config settings. self.login(username, password) self.database.generate_log_entries_for_settings() self.put_client_log_entries() self.get_server_log_entries() self.get_sync_finish() else: # Upload local changes and check for conflicts. result = self.put_client_log_entries() if result == "OK": self.put_client_media_files() self.get_server_media_files() self.get_server_log_entries() self.get_sync_finish() else: self.resolve_conflicts() self.ui.show_information("Sync finished!") except Exception, exception: self.ui.close_progress() serious = True if type(exception) == type(socket.gaierror()): self.ui.show_error("Could not find server!") serious = False elif type(exception) == type(socket.error()): self.ui.show_error("Could not connect to server!") serious = False elif type(exception) == type(socket.timeout()): self.ui.show_error("Timeout while waiting for server!") elif type(exception) == type(SyncError()): self.ui.show_error(str(exception)) serious = False elif type(exception) == type(SeriousSyncError()): self.ui.show_error(str(exception)) else: self.ui.show_error(traceback_string()) if serious and self.do_backup: # Only serious errors should result in the need for a full # sync next time. self.ui.show_error("Sync failed, restoring from backup. " + \ "The next sync will need to be a full sync.") if backup_file: self.database.restore(backup_file)
def put_login(self, environ): session = None try: self.ui.set_progress_text("Client logging in...") client_info_repr = environ["wsgi.input"].readline() client_info = self.text_format.parse_partner_info(\ client_info_repr) if not self.authorise(client_info["username"], client_info["password"]): self.ui.close_progress() return self.text_format.repr_message("Access denied") # Close old session waiting in vain for client input. # This will also close any session which timed out while # trying to log in just before, so we need to make sure the # client timeout is long enough for e.g. a NAS which is slow to # wake from hibernation. old_running_session_token = self.session_token_for_user.\ get(client_info["username"]) if old_running_session_token: self.terminate_session_with_token(old_running_session_token) session = self.create_session(client_info) # If the client database is empty, perhaps it was reset, and we # need to delete the partnership from our side too. if session.client_info["is_database_empty"] == True: session.database.remove_partnership_with(\ session.client_info["machine_id"]) # Make sure there are no cycles in the sync graph. Don't worry # about this if the database is empty, sinc the previous statement # would otherwise cause a spurious error. server_in_client_partners = self.machine_id in \ session.client_info["partners"] client_in_server_partners = session.client_info["machine_id"] in \ session.database.partners() if (server_in_client_partners and not client_in_server_partners)\ or \ (client_in_server_partners and not server_in_client_partners): if not session.client_info["is_database_empty"]: self.terminate_session_with_token(session.token) self.ui.close_progress() return self.text_format.repr_message("Sync cycle detected") # Detect the case where a user has copied the entire mnemosyne # directory before syncing. if session.client_info["machine_id"] == self.machine_id: self.terminate_session_with_token(session.token) self.ui.close_progress() return self.text_format.repr_message("same machine ids") # Create partnerships. session.database.create_if_needed_partnership_with(\ client_info["machine_id"]) session.database.merge_partners(client_info["partners"]) # Note that we need to send 'user_id' to the client as well, so # that the client can make sure the 'user_id's (used to label the # anonymous uploaded logs) are consistent across machines. server_info = {"user_id": session.database.user_id(), "machine_id": self.machine_id, "program_name": self.program_name, "program_version": self.program_version, "database_version": session.database.version, "partners": session.database.partners(), "session_token": session.token, "supports_binary_transfer": \ self.supports_binary_transfer(session), "is_database_empty": session.database.is_empty()} # Signal if we need a sync reset after restoring from a backup. server_info["sync_reset_needed"] = \ session.database.is_sync_reset_needed(\ client_info["machine_id"]) # Add optional program-specific information. server_info = \ session.database.append_to_sync_partner_info(server_info) return self.text_format.repr_partner_info(server_info)\ .encode("utf-8") except: # We need to be really thorough in our exception handling, so as # to always revert the database to its last backup if an error # occurs. It is important that this happens as soon as possible, # especially if this server is being run as a built-in server in a # thread in an SRS desktop application. # As mentioned before, the error handling should happen here, at # the lowest level, and not in e.g. 'wsgi_app'. return self.handle_error(session, traceback_string())