def check_login(browser, team: str, year: str) -> bool:
"""
Check if we're logged into iGEM websites.
Opens a random iGEM page and checks for edit access.
Arguments:
browser: mechanicalsoup.Browser instance
config: custom config dictionary
Returns:
True if we're logged in.
False otherwise.
"""
# Try opening a random page within the team
url = 'https://' + year + '.igem.org/Team:' + team + \
'/' + str(datetime.now().microsecond ** 2)
try:
browser.open(url)
except Exception:
message = "Couldn't connect to iGEM. Please check your internet connection."
logger.debug(message, exc_info=True)
logger.critical(message)
return False
soup = browser.get_current_page()
message = soup.text
# Check if we have edit access
if "do not have permission" in message:
return False
elif "edit this page" in message:
return True
return False
def iGEM_upload_page(browser, contents: str, url: str) -> bool:
"""
Uploads source code to the iGEM server.
Parameters:
browser: mechanicalsoup.Browser instance
contents: source code to be uploaded
url: the page where source code will uploaded
Returns:
True if successful, False otherwise.
"""
# Try opening the iGEM upload page
try:
browser.open(url) # TODO: Check this
except Exception:
message = "Lost connection to iGEM. Please check your internet connection."
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Select the form where source code has to be submitted.
# This might fail if the source code of the page changes.
try:
browser.select_form('form')
except Exception:
message = f"Couldn't find the form at {url}. Has the page changed?"
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Submit the form
browser['wpTextbox1'] = contents
try:
browser.submit_selected()
except Exception:
message = f"Couldn't upload to {url}."
logger.debug(message, exc_info=True)
logger.error(message)
return False
logger.info(f'Uploaded to {url}.')
return True
def iGEM_login(browser, credentials: dict, config: dict) -> bool:
"""
Logs into the iGEM server.
Arguments:
browser: mechanicalsoup.Browser instance
credentials: dictionary containing 'username'
and 'password'
config: custom configuration dictionary
Returns:
True if login is successful.
False along with an error message otherwise.
"""
# Check if we're already logged in
if check_login(browser, config['team'], config['year']):
logger.info("Already logged in.")
return True
# Try opening the login page
url = "https://igem.org/Login2"
try:
response = browser.open(url)
except Exception:
message = f"Couldn't connect to {url}."
logger.debug(message, exc_info=True)
logger.critical(message)
return False
# Check if login was successful
if response.status_code != 200:
message = f"Failed to login. {url} was not found."
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Select the form we have to fill.
# This might fail if the page changes.
try:
browser.select_form('form[method="post"]')
except Exception:
message = f"Couldn't find the login form at {url}. " + \
"Has the login page changed?"
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Fill the form
browser["username"] = credentials['username']
browser["password"] = credentials['password']
# Try submitting the form
try:
response = browser.submit_selected()
except Exception:
message = "Lost connection to iGEM servers."
logger.debug(message, exc_info=True)
logger.error(message)
return False
soup = BeautifulSoup(response.text, 'html5lib')
# Successful
if "successfully logged in" in soup.text:
logger.info(f"Successfully logged in as {credentials['username']}.")
return True
# Invalid username
elif "That username is not valid" in soup.text:
message = "This iGEM username is invalid."
logger.error(message)
# Invalid password
elif "That username is valid, but the password is not" in soup.text:
message = "This iGEM username is valid but the password is not."
logger.error(message)
# Unknown error
else:
message = "An unknown error occured while trying to login."
logger.error(message)
return False
def iGEM_upload_file(browser, file_object, year):
"""
Upload a file to iGEM servers.
iGEM allows files only 100MB large.
That check is performed in wikisync.run(), not here.
Parameters:
browser: mechanicalsoup.Browser instance
file_object: igem_wikisync.files.OtherFile object
Returns:
True if uploaded, False otherwise.
"""
# Try opening the iGEM upload page
url = file_object.upload_URL
try:
browser.open(url) # TODO: Check this
except Exception:
message = "Lost connection to iGEM. Please check your internet connection."
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Select the form where the file has to be uploaded.
# This might fail if the page changes.
try:
browser.select_form('form')
except Exception:
message = f"Couldn't find the form at {url}. Has the page changed?"
logger.debug(message, exc_info=True)
logger.error(message)
return False
browser['wpUploadFile'] = str(file_object.src_path)
browser['wpUploadDescription'] = 'Uploaded using WikiSync'
browser['wpDestFile'] = file_object.upload_filename
# * Ignore all warnings
# We keep track of already uploaded files internally
browser['wpIgnoreWarning'] = "1"
# Submit the form
try:
browser.submit_selected()
except Exception:
message = "Lost connection to iGEM servers."
logger.debug(message, exc_info=True)
logger.error(message)
return False
# Check whether there were any errors while uploading
return_url = browser.get_url()
if return_url == file_object.upload_URL:
message = "The following error occured while uploading " + file_object.upload_filename + ': '
message += browser.get_current_page().find(class_='error').text
logger.debug(message, exc_info=True)
logger.error(message)
return False
# TODO: Write test for this using KillSwitch svg from Ameya
# Extract relative link from response
print(str(file_object.src_path), file_object.upload_filename,
browser.get_url())
relative_link = browser.get_current_page().find(
class_='fullMedia').find('a')['href']
file_object.set_link_URL('https://' + year + '.igem.org' + relative_link)
logger.info(
f'Uploaded {file_object.upload_filename} to {file_object.link_URL}.')
return True
def upload_and_write_assets(other_files, browser, upload_map, config):
""""
Uploads and writes all files and stores URLs in upload_map.
Arguments:
other_files: dictionary containing OtherFile objects
browser: mechanicalsoup.StatefulBrowser instance
upload_map: custom upload map
config: custom configuration options
Returns:
Number of files uploaded
Raises:
SystemExit on failure
"""
# count the number of files uploaded
counter = 0
# files have to be uploaded before everything else because
# the URLs iGEM assigns are random
for path in other_files.keys():
file_object = other_files[path]
# flag to see if file has already been uploaded
uploaded = False
# check if the file has already been uploaded
for asset_path in upload_map['assets'].keys():
# if current path matches stored path
if asset_path == str(path):
asset = upload_map['assets'][asset_path]
# and the md5 hash is also the same
if file_object.md5_hash == asset['md5']:
# the file has already been uploaded
uploaded = True
break
else:
# the file path matches, but the md5 hash doesn't
# this means the file has changed
uploaded = False
break
# if new file
if not uploaded:
# write to build_dir
try:
# create directory if doesn't exist
if not os.path.isdir(file_object.build_path.parent):
os.makedirs(file_object.build_path.parent)
shutil.copyfile(
file_object.src_path, file_object.build_path.parent /
file_object.upload_filename)
except Exception:
# print upload map to save the current state
write_upload_map(upload_map)
message = f'Failed to write {str(file_object.path)} to build_dir. ' + \
'The current upload map has been saved. ' + \
'You will not have to upload everything again.'
logger.debug(message, exc_info=True)
logger.critical(message)
sys.exit(4)
successful = iGEM_upload_file(browser, file_object, config['year'])
if not successful:
# print upload map to save the current state
write_upload_map(upload_map)
message = f'Failed to upload {str(file_object.path)}. '
message += 'The current upload map has been saved. '
message += 'You will not have to upload everything again.'
logger.debug(message, exc_info=True)
logger.critical(message)
sys.exit(4)
else:
counter += 1
if str(path) in upload_map['assets'].keys():
upload_map['assets'][str(path)]['md5'] = file_object.md5_hash
upload_map['assets'][str(
path)]['link_URL'] = file_object.link_URL
else:
upload_map['assets'][str(path)] = {
'link_URL': file_object.link_URL,
'md5': file_object.md5_hash,
'upload_filename': file_object.upload_filename
}
return counter
def cache_files(upload_map, config):
"""
Loads filenames into memory, along with setting up
appropriate objects to generate URLs and hashes as required.
Arguments:
upload_map: custom upload map
config: configuration for this run
Returns:
cache: dictionary with html, css, js and other file objects
"""
cache = {'html': {}, 'css': {}, 'js': {}, 'other': {}}
# for each file in src_dir
for root, _, files in os.walk(config['src_dir']):
for filename in files:
# Store path and extension
infile = (Path(root) / Path(filename)).relative_to(
config['src_dir'])
extension = infile.suffix[1:].lower()
# create appropriate file object
# file objects contain corresponding paths and
if extension in ['html', 'css', 'js']:
file_object = None
if extension == 'html':
file_object = HTMLfile(infile, config)
elif extension == 'css':
file_object = CSSfile(infile, config)
elif extension == 'js':
file_object = JSfile(infile, config)
# In poster mode, make sure URL starts with /Poster after team
if config['poster_mode']:
link_URL = file_object.link_URL
after_team = link_URL.split(config['team'])[1]
if len(after_team) < 7 or after_team[0:7] != "/Poster":
message = 'All files must start with /Poster in poster mode.'
logger.debug(message, exc_info=True)
logger.critical(message)
raise Exception
cache[extension][file_object.path] = file_object
elif extension.lower() in [
'png', 'gif', 'jpg', 'jpeg', 'pdf', 'ppt', 'txt', 'zip',
'mp3', 'mp4', 'webm', 'mov', 'swf', 'xls', 'xlsx', 'docx',
'pptx', 'csv', 'm', 'ogg', 'gb', 'tif', 'tiff', 'fcs',
'otf', 'eot', 'ttf', 'woff', 'svg'
]:
# make sure file path start with 'assets'
if len(str(infile)) < 7 or infile.parts[0] != 'assets':
logger.error(
f'{infile} is an {extension} file outside the "assets" folder. Skipping.'
)
continue
# make sure file size is within limits
elif (config['src_dir'] / infile).stat().st_size >= 100000000:
logger.error(
f'{infile} is larger than the 100MB file limit. Skipping.'
)
continue
# create OtherFile
else:
file_object = OtherFile(infile, config)
if len(file_object.upload_filename) < 240:
cache['other'][file_object.path] = file_object
else:
logger.error(
f'{infile}: Upload filename too large. Skipping.')
logger.error(
'Please do not nest assets too deep and take a look at our docs to see how WikiSync renames files.'
)
continue
else:
logger.error(
f'{infile} has an unsupported file extension. Skipping.')
continue
if extension in ['html', 'css', 'js']:
if str(file_object.path) not in upload_map[extension].keys():
upload_map[extension][str(file_object.path)] = {
'md5': '',
'link_URL': file_object.link_URL
}
return cache