def iGEM_upload_page(browser, contents: str, url: str) -> bool:
"""
Uploads source code to the iGEM server.
Parameters:
browser: mechanicalsoup.Browser instance
contents: source code to be uploaded
url: the page where source code will uploaded
Returns:
True if successful, False otherwise.
"""
# Try opening the iGEM upload page
try:
browser.open(url) # TODO: Check this
except Exception:
message = "Lost connection to iGEM. Please check your internet connection."
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Select the form where source code has to be submitted.
# This might fail if the source code of the page changes.
try:
browser.select_form('form')
except Exception:
message = f"Couldn't find the form at {url}. Has the page changed?"
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Submit the form
browser['wpTextbox1'] = contents
try:
browser.submit_selected()
except Exception:
message = f"Couldn't upload to {url}."
logger.debug(message, exc_info=True)
logger.error(message)
return False
logger.info(f'Uploaded to {url}.')
return True
def iGEM_URL(config: dict, path: Path, upload_map: dict, url: str) -> str:
"""
Replaces a given absolute local URL with it's iGEM counterpart.
Arguments:
config: Dictionary containing 'src_dir', 'team' and 'build_dir'.
path: path to the file where this URL was found
upload_map: custom upload map
url: the absolute path to be converted
Returns:
URL where this file would be found on iGEM servers.
Returns false if URL with an unsupported extension is passed
"""
if config['silence_warnings']:
logger.handlers[0].setLevel(40)
# Store input for logging and/or returning
old_path = url
# Convert to path in case a string was passed
path = Path(path)
# return if it's already absolute
if not is_relative(url):
return url
if url == '/':
return 'https://' + config['year'] + '.igem.org/Team:' + config['team']
# Resolve relative path to local absolute path
resolved_path = resolve_relative_path(url, path.parent, config['src_dir'])
# check upload_map
found = False
for filetype in upload_map.keys():
if str(resolved_path) in upload_map[filetype].keys():
url = upload_map[filetype][str(resolved_path)]['link_URL']
found = True
break
if not found:
# check if file exists
filepath = config['src_dir'] / resolved_path
if not os.path.isfile(filepath):
message = f"{filepath} is referenced in {config['src_dir'] / path} but was not found."
logger.warning(message)
extension = resolved_path.suffix[1:].lower()
# create imaginary file object and
# let the functions in that class handle creating URLs
if extension == 'html':
file_object = HTMLfile(resolved_path, config)
url = file_object.link_URL
elif extension == 'css':
file_object = CSSfile(resolved_path, config)
url = file_object.link_URL
elif extension == 'js':
file_object = JSfile(resolved_path, config)
url = file_object.link_URL
# leave unchanged
else:
logger.warning(
f"{old_path} is referenced in {path} but was not found.")
return old_path
logger.info(f"{old_path} was changed to {url} in {path}.")
return url
def iGEM_login(browser, credentials: dict, config: dict) -> bool:
"""
Logs into the iGEM server.
Arguments:
browser: mechanicalsoup.Browser instance
credentials: dictionary containing 'username'
and 'password'
config: custom configuration dictionary
Returns:
True if login is successful.
False along with an error message otherwise.
"""
# Check if we're already logged in
if check_login(browser, config['team'], config['year']):
logger.info("Already logged in.")
return True
# Try opening the login page
url = "https://igem.org/Login2"
try:
response = browser.open(url)
except Exception:
message = f"Couldn't connect to {url}."
logger.debug(message, exc_info=True)
logger.critical(message)
return False
# Check if login was successful
if response.status_code != 200:
message = f"Failed to login. {url} was not found."
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Select the form we have to fill.
# This might fail if the page changes.
try:
browser.select_form('form[method="post"]')
except Exception:
message = f"Couldn't find the login form at {url}. " + \
"Has the login page changed?"
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Fill the form
browser["username"] = credentials['username']
browser["password"] = credentials['password']
# Try submitting the form
try:
response = browser.submit_selected()
except Exception:
message = "Lost connection to iGEM servers."
logger.debug(message, exc_info=True)
logger.error(message)
return False
soup = BeautifulSoup(response.text, 'html5lib')
# Successful
if "successfully logged in" in soup.text:
logger.info(f"Successfully logged in as {credentials['username']}.")
return True
# Invalid username
elif "That username is not valid" in soup.text:
message = "This iGEM username is invalid."
logger.error(message)
# Invalid password
elif "That username is valid, but the password is not" in soup.text:
message = "This iGEM username is valid but the password is not."
logger.error(message)
# Unknown error
else:
message = "An unknown error occured while trying to login."
logger.error(message)
return False
def iGEM_upload_file(browser, file_object, year):
"""
Upload a file to iGEM servers.
iGEM allows files only 100MB large.
That check is performed in wikisync.run(), not here.
Parameters:
browser: mechanicalsoup.Browser instance
file_object: igem_wikisync.files.OtherFile object
Returns:
True if uploaded, False otherwise.
"""
# Try opening the iGEM upload page
url = file_object.upload_URL
try:
browser.open(url) # TODO: Check this
except Exception:
message = "Lost connection to iGEM. Please check your internet connection."
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Select the form where the file has to be uploaded.
# This might fail if the page changes.
try:
browser.select_form('form')
except Exception:
message = f"Couldn't find the form at {url}. Has the page changed?"
logger.debug(message, exc_info=True)
logger.error(message)
return False
browser['wpUploadFile'] = str(file_object.src_path)
browser['wpUploadDescription'] = 'Uploaded using WikiSync'
browser['wpDestFile'] = file_object.upload_filename
# * Ignore all warnings
# We keep track of already uploaded files internally
browser['wpIgnoreWarning'] = "1"
# Submit the form
try:
browser.submit_selected()
except Exception:
message = "Lost connection to iGEM servers."
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Check whether there were any errors while uploading
return_url = browser.get_url()
if return_url == file_object.upload_URL:
message = "The following error occured while uploading " + file_object.upload_filename + ': '
message += browser.get_current_page().find(class_='error').text
logger.debug(message, exc_info=True)
logger.error(message)
return False
# TODO: Write test for this using KillSwitch svg from Ameya
# Extract relative link from response
print(str(file_object.src_path), file_object.upload_filename,
browser.get_url())
relative_link = browser.get_current_page().find(
class_='fullMedia').find('a')['href']
file_object.set_link_URL('https://' + year + '.igem.org' + relative_link)
logger.info(
f'Uploaded {file_object.upload_filename} to {file_object.link_URL}.')
return True
def build_and_upload(files, browser, config, upload_map):
"""
Replaces URLs in files and uploads changed files.
Arguments:
files: Custom file cache
browser: mechanicalsoup.StatefulBrowser instance
config: Configuration for this run
upload_map: custom upload map
Returns:
Dictionary with no. of 'html', 'css' and 'js' files uploaded
"""
counter = {
'html': 0,
'css': 0,
'js': 0,
}
for file_dictionary in [files['html'], files['css'], files['js']]:
for path in file_dictionary.keys():
file_object = file_dictionary[path]
path_str = str(file_object.path)
ext = file_object.extension
# open file
try:
with open(file_object.src_path, 'r') as file:
contents = file.read()
except Exception:
message = f'Could not open/read {file_object.path}. Skipping.'
logger.error(message)
continue # FIXME Can this be improved?
processed = None # just so the linter doesn't freak out
# parse and modify contents
if ext == 'html':
processed = HTMLparser(config, file_object.path, contents,
upload_map)
elif ext == 'css':
processed = CSSparser(config, file_object.path, contents,
upload_map)
elif ext == 'js':
processed = JSparser(contents)
# calculate and store md5 hash of the modified contents
build_hash = md5(processed.encode('utf-8')).hexdigest()
if upload_map[ext][path_str]['md5'] == build_hash:
message = f'Contents of {file_object.path} have been uploaded previously. Skipping.'
logger.info(message)
else:
upload_map[ext][path_str]['md5'] = build_hash
build_path = file_object.build_path
try:
# create directory if doesn't exist
if not os.path.isdir(build_path.parent):
os.makedirs(build_path.parent)
# and write the processed contents
with open(build_path, 'w') as file:
file.write(processed)
except Exception:
message = f"Couldn not write {str(file_object.build_path)}. Skipping."
logger.error(message)
continue
# FIXME Can this be improved?
# upload
successful = iGEM_upload_page(browser, processed,
file_object.upload_URL)
if not successful:
message = f'Could not upload {str(file_object.path)}. Skipping.'
logger.error(message)
continue
# FIXME Can this be improved?
else:
counter[ext] += 1
return counter