def UnzipData(zipped_data): """Unzip the specified data using a temporary file and the gzip library. Args: A byte array with the zipped data. Returns: A byte array with the unzipped data. Raises: error.InternalError: If any I/O or Unicode error occurs while unzipping the data. """ # Uncompress the zipped data using a memory-mapped file. try: zipped_memfile = StringIO.StringIO(zipped_data) unzipped_file = gzip.GzipFile(fileobj=zipped_memfile, mode='rb') unzipped_data = unzipped_file.read() unzipped_file.close() zipped_memfile.close() return unzipped_data except UnicodeError as e: raise error.InternalError('Unicode error while parsing zipped data: ' '{0}.\n'.format(e)) except IOError as e: raise error.InternalError( 'I/O error while decompressing unzipped data: ' '{0}.\n'.format(e))
def ZipData(unzipped_data): """Zip the specified data using a temporary file and the gzip library. Args: A byte array with the unzipped data. Returns: A byte array with the zipped data. Raises: error.InternalError: If any I/O or Unicode error occurs while unzipping the data. """ try: # Compress the unzipped data into a memory-mapped file. zipped_memfile = StringIO.StringIO() zipped_file = gzip.GzipFile(fileobj=zipped_memfile, mode='wb') zipped_file.write(unzipped_data) zipped_file.close() # Extract the zipped data from the memory-mapped file, release it and # return the zipped data. zipped_data = zipped_memfile.getvalue() zipped_memfile.close() return zipped_data except UnicodeError as e: raise error.InternalError('Unicode error while parsing zipped data: ' '{0}.\n'.format(e)) except IOError as e: raise error.InternalError( 'I/O error while decompressing unzipped data: ' '{0}.\n'.format(e))
def _ReadDataImpl(filename): """Read the persistent data from the specified file, which should be formatted as a python dict. Args: filename: Name of the file with the data to load. Returns: A python dictionary with the file contents. Raises: error.InternalError: If there is any error while reading the data from the file. """ # Open the specified file and get its contents. Then evaluate the file data # directly, as it is formatted as a python dict. try: file = open(filename, 'rt') file_data = file.read() file.close() return eval(file_data, {}, {}) except IOError as e: raise error.InternalError( 'IO error happened while reading data from file ' '"{0}" : {1}.\n'.format(filename, e))
def ReadData(default_data=None): """Read tool configuration data. Args: default_data: Dictionary with default values for fields that do not appear in both configuration files. Returns: A dictionary with the validated configuration data. Raises: error.InternalError: If there is any problem while reading the user configuration file. """ # Read the user data and check if it was read successfully. try: user_data = _ReadDataImpl(constants.USER_CONFIG_FILE) except error.InternalError: raise error.InternalError('Unable to read used data.\n') # Read the current contest data and show warning if not read successfully. try: current_data = _ReadDataImpl(constants.CURRENT_CONFIG_FILE) except error.InternalError: sys.stderr.write('Warning: Cannot read current data.\n') current_data = {} # Start with the default data, patch it with the current data and then with # the user's data, so the latter ones have more priority. Then validate and # return the data. final_data = dict(default_data or _DEFAULT_DATA) final_data.update(current_data) final_data.update(user_data) _ValidateData(final_data) return final_data
def AddFile(self, name, filename): """Add a file's contents to this multipart data. Args: name: Name of the element to add to the multipart data. filename: Name of the file with the contents to add to the multipart data. Raises: error.InternalError: If a problem occurs when reading the file. """ try: # Read the data from the specified file. file = open(filename, 'rb') file_data = file.read() file.close() # Append the metadata and then the read file data. Finally, complete with # a closing boundary. self.data.append('--' + self.boundary) self.data.append('Content-Disposition: form-data; name="{0}"; ' 'filename="{1}"'.format(name, filename)) self.data.append('Content-Type: {0}'.format( self._GetContentType(filename))) self.data.append('') self.data.append(file_data) except IOError as e: raise error.InternalError('I/O error while reading file "{0}": ' '{1}.\n'.format(filename, e))
def AddFile(self, name, filename, file_data=None): """Add a file's contents to this multipart data. Args: name: Name of the element to add to the multipart data. filename: Name of the file with the contents to add to the multipart data. file_data: Data of the file to be added. If None, the data will be read from this file instead. This parameter is here to support memory-mapped files. Raises: error.InternalError: If a problem occurs when reading the file. """ # Read the data from the specified file. if file_data is None: try: file_obj = open(filename, 'rb') file_data = file_obj.read() file_obj.close() except IOError as e: raise error.InternalError( 'I/O error while reading file "{0}": ' '{1}.\n'.format(filename, e)) # Append the metadata and then the read file data. Finally, complete with # a closing boundary. self.data.append('--' + self.boundary) self.data.append('Content-Disposition: form-data; name="{0}"; ' 'filename="{1}"'.format(name, filename)) self.data.append('Content-Type: {0}'.format( self._GetContentType(filename))) self.data.append('') self.data.append(file_data)
def _ReadDataImpl(filename): """Read the persistent data from the specified file, which should be formatted as a python dict. Args: filename: Name of the file with the data to load. Returns: A python dictionary with the file contents. Raises: error.InternalError: If there is any error while reading the data from the file. """ try: # Open the configuration file and read all lines from it. file = open(filename, 'rt') file_lines = file.readlines() file.close() # Prepare the file data to prevent mistakes and evaluate it as if it were a # python dictionary. file_data = _PrepareFileData(file_lines) return eval(file_data, {}, {}) except IOError as e: raise error.InternalError('IO error happened while reading data from file ' '"{0}" : {1}.\n'.format(filename, e))
def MakeZipFileInMemory(source_files, ignore_exts=None): """Create a zip file with the specified source files in memory. The source_files sequence can include directories, which will be traversed and added recursively to the output file, ignoring those with banned extensions. Args: source_files: A collection with all source files or directories to zip. ignore_exts: A collection with all the extensions to ignore. Returns: A (zip_output, ignored_files) tuple, where zip_output are the contents of the generated zip file and ignored_files is a list with all the ignored files during the zip file creation. Raises: error.InternalError: If any Unicode error occurs while zipping the data. """ # Create a memory-mapped file and create the zip file on it. Then, get its # contents, close the file and return. try: output_file = StringIO.StringIO() ignored_files = MakeZipFile(source_files, output_file, ignore_exts) zip_output = output_file.getvalue() output_file.close() return zip_output, ignored_files except UnicodeError as e: raise error.InternalError('Unicode error while parsing zipped data: ' '{0}.\n'.format(e))
def _WriteDataImpl(data, filename): """Write the specified data to the specified file, which will be formatted as a python dict. Args: data: Python dictionary with the data to write to the file. filename: Name of the file where the data should be written. Raises: error.InternalError: If there is any error while writing the data to the file. """ try: # Calculate the space needed for the keys and create a format string # for each data item. key_width = max(len(repr(key)) for key in data.iterkeys()) item_format = '{0:%d} : {1},' % key_width # Open the file and store each item inside it. file = open(filename, 'wt') file.write('{\n') for key, value in sorted(data.iteritems()): item_line = item_format.format(repr(key), repr(value)) file.write('{0}\n'.format(item_line)) file.write('}\n') file.close() except IOError as e: raise error.InternalError( 'IO error happened while writing data to file ' '"{0}" : {1}.\n'.format(filename, e))
def ZipData(unzipped_data): """Zip the specified data using a temporary file and the gzip library. After the data is zipped, the temporary file is erased, so no special cleanup is necessary. Args: A byte array with the unzipped data. Returns: A byte array with the zipped data. Raises: error.InternalError: If any I/O or OS error occurs while unzipping the data. """ # Compress the data and write it to a temporary file (using a random name to # prevent collisions). try: zip_filename = 'tempZipFile_{0}.gz'.format( random.randrange(0, 2**31 - 1)) compress_file = gzip.open(zip_filename, 'wb') compress_file.write(unzipped_data) compress_file.close() except IOError as e: raise error.InternalError( 'I/O error while compressing data into "{0}": ' '{1}.\n'.format(zip_filename, e)) # Open the file normally and get the zipped contents. try: zipped_file = open(zip_filename, 'rb') zipped_data = zipped_file.read() zipped_file.close() except IOError as e: raise error.InternalError('I/O error while reading unzipped data from ' '"{0}": {1}.\n'.format(zip_filename, e)) # Remove the temporary zipped file. try: os.remove(zip_filename) except OSError as e: raise error.InternalError( 'OS error happened while removing zipped data at ' '"{0}": {1}.\n'.format(zip_filename, e)) # Return the zipped string. return zipped_data
def ClearContest(): # Erase the current configuration file if it exists and is a file, otherwise # show a warning if the configuration is not a regular file (should not happen # under normal conditions). try: if os.path.isfile(constants.CURRENT_CONFIG_FILE): os.remove(constants.CURRENT_CONFIG_FILE) elif os.path.exists(constants.CURRENT_CONFIG_FILE): sys.stderr.write( 'Warning: Cannot erase current configuration file "{0}" ' 'because it is not a regular file.\n'.format( constants.CURRENT_CONFIG_FILE)) except OSError as e: raise error.InternalError( 'OS error happened while deleting file "{0}": ' '{1}.\n'.format(filename, e))
def WriteData(data): """Write tool configuration data into the current file. Args: data: Dictionary with tool configuration. Raises: error.InternalError: If there is any problem while writing the current configuration file. """ # Just forward the data to the current configuration file, the user # configuration should not be changed by the tool. try: _WriteDataImpl(data, constants.CURRENT_CONFIG_FILE) except error.InternalError: raise error.InternalError('Cannot write data to the current ' 'configuration.\n')
def Initialize(contest_id, password=None): """Initialize configuration for the specified contest, storing the retrieved data in the current configuration file. Args: contest_id: ID of the contest to initialize. password: Password specified by the user, if any. """ # Reset the current configuration file with the one provided by the user and # renew the cookie, so the middleware tokens are retrieved correctly. try: shutil.copy(constants.USER_CONFIG_FILE, constants.CURRENT_CONFIG_FILE) code_jam_login.Login(password) except OSError as e: raise error.InternalError( 'Configuration file {0} could not be created: ' '{1}.\n'.format(constants.CURRENT_CONFIG_FILE, e)) # Read the current configuration file and extract the host and the cookie. try: contest_data = data_manager.ReadData() host = contest_data['host'] cookie = contest_data['cookie'] except KeyError as e: # Indicate that no host or cookie was configured and exit with error. raise error.ConfigurationError('No host or login cookie found in the ' 'configuration file: {0}.\n'.format(e)) # Retrieve the problem list and validate the extracted contest data and exit # if there is any error. problems = _GetProblems(host, cookie, contest_id) _ValidateContestDataOrRaise(contest_data['middleware_tokens'], problems) # Add the contest id, the problems and the middleware tokens to the contest # data. contest_data['contest_id'] = contest_id contest_data['problems'] = problems # Finally, write the contest data to the current data file, and then # renew the cookie stored in the configuration. data_manager.WriteData(contest_data) sys.stdout.write('Contest {0} initialized successfully, {1} problem(s) ' 'retrieved.\n'.format(contest_id, len(problems)))
def Download(self, input_id, filename): """Download the specified input and store it in the specified file. Args: input_id: Identifier of the input to download ('0' for the small input and '1' for the large input). filename: Name of the file where the input data must be stored. """ # Send an HTTP request to get the input file from the server. sys.stdout.write('Getting input file "{0}" from "{1}"...\n'.format( filename, self.host)) request_url = '/codejam/contest/dashboard/do/{0}'.format(filename) request_referer = 'http://{0}/codejam/contest/dashboard?c={1}'.format( self.host, self.contest_id) request_arguments = { 'cmd': 'GetInputFile', 'contest': self.contest_id, 'problem': str(self.problem_id), 'input_id': input_id, 'filename': filename, 'input_file_type': '0', 'csrfmiddlewaretoken': self.middleware_token, 'agent': constants.CODEJAM_AGENT_NAME, } request_headers = { 'Referer': request_referer, 'Cookie': self.cookie, } try: status, reason, response = http_interface.Get( self.host, request_url, request_arguments, request_headers) except httplib.HTTPException as e: raise error.NetworkError( 'HTTP exception while getting input file: ' '{0}.\n'.format(e)) # Check if the status is not good. if status != 200 or reason != 'OK': raise error.ServerError( 'Error while communicating with the server, ' 'cannot download input. Check that the host, ' 'username and contest id are valid.\n') # No response from the server, output warning. if not response: raise error.ServerError( 'No response received from the server. This generally happens when:\n' ' - You try to download a small input but it is already solved.\n' ' - You try to download the large input before solving the small ' 'input.\n' ' - You try to redownload the large but its timer already ' 'expired.\n') # Write response to the desired file. try: input_file = open(filename, 'wt') input_file.write(response) input_file.close() sys.stdout.write( 'File "{0}" downloaded successfully.\n'.format(filename)) except IOError as e: raise error.InternalError('I/O error while writing file "{0}": ' '{1}.\n'.format(filename, e))
def CheckToolVersion(host, tool_version): """Check if the specified tool_version is accepted by the host. Args: host: Host where the contest is running. tool_version: String with this tool's version. Raises: error.InternalError: If the tool's version is not accepted by the host. error.NetworkError: If a network error occurs while communicating with the server. error.ServerError: If the server answers code distinct than 200 or the response is a malformed JSON. """ # Send an HTTP request to get the problem list from the server. sys.stdout.write('Checking tool version for "{0}"...\n'.format(host)) request_referer = 'http://{0}/codejam'.format(host) request_arguments = { 'cmd': 'CheckVersion', 'version': tool_version, } request_headers = { 'Referer': request_referer, } try: status, reason, response = http_interface.Get(host, '/codejam/cmdline', request_arguments, request_headers) except httplib.HTTPException as e: raise error.NetworkError( 'HTTP exception while checking tool version with ' 'the Google Code Jam server: {0}\n'.format(e)) # The current server version might not support this yet. Print a warning # message and skip validation. if status == 404: print 'WARNING: Cannot check commandline version with the server.' return # Check if the status is not good. if status != 200 or reason != 'OK': raise error.ServerError( 'Error while communicating with the server, cannot ' 'check tool version. Check that the host is ' 'valid.\n') # Extract token information from server response. try: validation_info = json.loads(response) if validation_info['msg']: print validation_info['msg'] if not validation_info['valid']: raise error.InternalError( 'This tool\'s version is not accepted by host ' '{0}, please update to the latest ' 'version.\n'.format(host, tool_version)) except (KeyError, ValueError) as e: raise error.ServerError( 'Invalid response received from the server, cannot ' 'initialize contest. Check that the contest id is ' 'valid: {0}.\n'.format(e))
def Initialize(tournament_id, contest_id, password=None): """Initialize configuration for the specified tournament or contest. This function initializes the tool for a contest. If the contest is None, the tool will be initialized for the current contest of the specified tournament. Either one of tournament_id or contest_id must be not None. The retrieved data is stored in the current configuration file. Args: tournament_id: ID of the tournament whose current contest must be initialized. contest_id: ID of the contest to initialize. If None, the server will ask for the current contest of the specified tournament. password: Password specified by the user, if any. Raises: error.ConfigurationError: If the contest data is invalid or incomplete. error.UserError: If no contest was specified and there is no running contest for the specified tournament. """ # Reset the current configuration file with the one provided by the user and # renew the cookie, so the middleware tokens are retrieved correctly. try: user_config_path = data_manager.ParametrizeConfigPath( constants.USER_CONFIG_PATH) current_config_path = data_manager.ParametrizeConfigPath( constants.CURRENT_CONFIG_PATH) shutil.copy(user_config_path, current_config_path) code_jam_login.Login(password) except OSError as e: raise error.InternalError( 'Configuration file {0} could not be created: ' '{1}.\n'.format(current_config_path, e)) # Read the current configuration file and extract the host and the cookie. try: contest_data = data_manager.ReadData() host = contest_data['host'] cookie = contest_data['cookie'] except KeyError as e: # Indicate that no host or cookie was configured and exit with error. raise error.ConfigurationError('No host or login cookie found in the ' 'configuration file: {0}.\n'.format(e)) # Get the current contest if no contest id was specified. If there is no # running contest show an error to the user. if contest_id is None: contest_id = GetCurrentContestId(host, cookie, tournament_id) if contest_id is None: raise error.UserError( 'No contest is running for tournament %s and no ' 'contest id was specified.\n' % tournament_id) sys.stdout.write( 'Initializing tool for current contest with id %s.\n' % contest_id) # Retrieve the problem list and validate the extracted contest data and exit # if there is any error. problems = _GetProblems(host, cookie, contest_id) _ValidateContestDataOrRaise(contest_data['middleware_tokens'], problems) # Add the contest id, the problems and the middleware tokens to the contest # data. contest_data['contest_id'] = contest_id contest_data['problems'] = problems # Finally, write the contest data to the current data file, and then # renew the cookie stored in the configuration. data_manager.WriteData(contest_data) sys.stdout.write('Contest {0} initialized successfully, {1} problem(s) ' 'retrieved.\n'.format(contest_id, len(problems)))
def Submit(self, input_id, output_name, source_patterns, io_set_public, gzip_body=True, zip_sources=False, add_ignored_zips=False): """Submit the specified output and sources file to the problem. Args: input_id: Identifier of the output to submit ('0' for the small output, '1' for the large output). output_name: Name of the file with the output data. source_patterns: Name patterns of the source files to be included with the output. These patterns will be expanded using Python's glob module. io_set_public: Boolean indicating whether the answer is public or not. gzip_body: Boolean indicating whether the body has to be gzipped or not. zip_sources: Boolean indicating whether all sources should be put inside a zip file or not. add_ignored_zips: Boolean indicating whether zip files that are not included directly but are inside a included directory should be submitted or not. Raises: error.InternalError: If an error occurs while copying ignored zip files to the final location. error.NetworkError: If a network error occurs while communicating with the server. error.ServerError: If the server answers code distinct than 200. """ # Prepare the source files (zipping all directories). After this, # source_files will only contain text files and zip files specified directly # or by compressing a directory. source_files, ignored_zips = self._PrepareSourceFiles( set(source_patterns)) # Check if the user requested to zip source files. if zip_sources: # Extract all files to zip into a list and remove them from the # source files list. sources_to_zip = [ filename for filename, file_data in source_files if file_data is None ] source_files = [(filename, file_data) for filename, file_data in source_files if file_data is not None] # Generate a zip file with all the flat source files. sys.stdout.write('Compressing files "{0}"...\n'.format( ', '.join(sources_to_zip))) zipped_sources, _ = zip_utils.MakeZipFileInMemory( sources_to_zip, ignore_exts=['.zip']) # Generate a random name for the zipped source files and add it to the # source files to send. zip_filename = '__plain_files_{0}.zip'.format( random.randrange(0, 2**31 - 1)) source_files.append((zip_filename, zipped_sources)) # Check if the user requested to add the ignored zip files inside included # directories. if add_ignored_zips: # Copy and add each ignored file to the source list. for ignored_zip in ignored_zips: # Read the contents of the ignored zip file. try: zip_file = open(ignored_zip, 'rb') zip_file_contents = zip_file.read() zip_file.close() except IOError as e: raise error.InternalError( 'I/O error happened while read zip file ' '"{0}": {1}.\n'.format(ignored_zip, e)) # Generate the zip flat filename by substituting path with underscores # and adding a random number to prevent collisions. Then use it to add # the zip file and its contents into the source files to send. path, ext = os.path.splitext(ignored_zip) zip_flat_filename = '{1}_{0}{2}'.format( random.randrange(0, 2**31 - 1), path.replace('\\', '_').replace('/', '_'), ext) source_files.append((zip_flat_filename, zip_file_contents)) # Print message and check that at least one source file was included. if source_files: sources_str = ', '.join('"{0}"'.format(filename) for filename, _ in source_files) sys.stdout.write( 'Sending output file "{0}" and source(s) {1} to "{2}"' '...\n'.format(output_name, sources_str, self.host)) else: # Print warning saying that no source file is being included, this might # cause disqualification in a real contest. sys.stdout.write( 'Warning, no source files are being sent for output ' 'file "{0}"!\n'.format(output_name)) sys.stdout.write('Sending output file "{0}" to "{1}"...\n'.format( output_name, self.host)) # Generate a random boundary string to separate multipart data and # create a multipart data object with it. Then fill it with the necessary # arguments. multipart_boundary = '----gcjMultipartBoundary{0}'.format( random.randrange(0, 2**31 - 1)) body_data = multipart_data.MultipartData(multipart_boundary) body_data.AddString('csrfmiddlewaretoken', self.middleware_token) body_data.AddFile('answer', output_name) for i, (filename, file_data) in enumerate(source_files): body_data.AddFile('source-file{0}'.format(i), filename, file_data) body_data.AddString('source-file-name{0}'.format(i), filename) body_data.AddString('cmd', 'SubmitAnswer') body_data.AddString('contest', self.contest_id) body_data.AddString('problem', self.problem_id) body_data.AddString('input_id', input_id) body_data.AddString('num_source_files', str(len(source_files))) body_data.AddString('agent', constants.CODEJAM_AGENT_NAME) # Get the message body and check if compression was requested. request_body = str(body_data) if gzip_body: compressed_body = zip_utils.ZipData(request_body) sys.stdout.write('Sending {0} bytes to server ({1} uncompressed)' '...\n'.format(len(compressed_body), len(request_body))) request_body = compressed_body else: sys.stdout.write('Sending {0} bytes to server...\n'.format( len(request_body))) # Send an HTTP request with the output file and the source files. request_url = '/codejam/contest/dashboard/do' request_referer = 'http://{0}/codejam/contest/dashboard?c={1}'.format( self.host, self.contest_id) request_arguments = {} request_headers = { 'Content-Encoding': 'gzip' if gzip_body else 'text/plain', 'Content-Type': 'multipart/form-data; boundary={0}'.format(multipart_boundary), 'Referer': request_referer, 'Cookie': self.cookie, } try: status, reason, response = http_interface.Post( self.host, request_url, request_arguments, request_headers, request_body) except httplib.HTTPException as e: raise error.NetworkError( 'HTTP exception while sending solution to the ' 'Google Code Jam server: {0}.\n'.format(e)) # Check if the status is not good. if status != 200 or reason != 'OK': raise error.ServerError( 'Error while communicating with the server, ' 'cannot submit solution. Check that the host, ' 'username and contest id are valid.\n') # Check if the server accepted the input or just ignored it. If it # accepted it, parse the response and print the submission result. if response: submit_result = self._ParseResult(response, io_set_public) sys.stdout.write('{0}\n'.format(submit_result)) else: raise error.ServerError( 'No response received from the server. This generally happens if ' 'you try to submit the large output before solving the small ' 'input.\n')
def MakeZipFile(source_files, output_file, ignore_exts=None): """Create a zip file with the specified source files. The source_files sequence can include directories, which will be traversed and added recursively to the output file, ignoring those with banned extensions. Args: source_files: A collection with all source files or directories to zip. output_file: Name or file-like object where the zip file must be generated. ignore_exts: A collection with all the extensions to ignore. Returns: A list with all the ignored files during the zip file creation. Raises: error.InternalError: If any I/O or OS error occurs while zipping the data. """ if ignore_exts is None: ignore_exts = [] try: # Open the destination zip file and initialize the ignored files set. zip_file = zipfile.ZipFile(output_file, 'w', zipfile.ZIP_DEFLATED) ignored_files = set() # Put all specified sources in the zip file, ignoring files with the # specified extensions. for source_filename in source_files: # If the source is a directory, walk over it, adding each file inside it. if os.path.isdir(source_filename): # Walk over the specified directory. for dirpath, dirnames, filenames in os.walk(source_filename): # Create the directory inside the zip file and process all # files in the current directory. zip_file.write(dirpath) for filename in filenames: # Create the base filename and check if it extension is not in the # extenstions ignore list. Otherwise, add it to the ignored files # set. base_filename = os.path.join(dirpath, filename) if os.path.splitext(filename)[1] not in ignore_exts: zip_file.write(base_filename) else: ignored_files.add(base_filename) else: # Add a file to the zip if and only if it extension is not in the # ignore list. Otherwise, add it to the ignored files set. if os.path.splitext(source_filename)[1] not in ignore_exts: zip_file.write(source_filename) else: ignored_files.add(source_filename) # Close the zip file and return the ignored files set. zip_file.close() return ignored_files except IOError as e: raise error.InternalError('I/O error while creating zip file: ' '{0}.\n'.format(e)) except OSError as e: raise error.InternalError('OS error while creating zip file: ' '{0}.\n'.format(e))