async def get_dailies(session: ClientSession, header: dict,
wfwx_stations: List[WFWXWeatherStation],
time_of_interest: datetime) -> List[dict]:
""" Get the daily actuals/forecasts for the given station ids. """
# build a list of wfwx station id's
wfwx_station_ids = [wfwx_station.wfwx_id for wfwx_station in wfwx_stations]
timestamp_of_interset = math.floor(time_of_interest.timestamp() * 1000)
# for local dev, we can use redis to reduce load in prod, and generally just makes development faster.
# for production, it's more tricky - we don't want to put too much load on the wf1 api, but we don't
# want stale values either. We default to 5 minutes, or 300 seconds.
cache_expiry_seconds = config.get(
'REDIS_DAILIES_BY_STATION_CODE_CACHE_EXPIRY', 300)
use_cache = cache_expiry_seconds is not None and config.get(
'REDIS_USE') == 'True'
dailies_iterator = fetch_paged_response_generator(
session,
header,
BuildQueryDailiesByStationCode(timestamp_of_interset,
timestamp_of_interset,
wfwx_station_ids),
'dailies',
use_cache=use_cache,
cache_expiry_seconds=cache_expiry_seconds)
return dailies_iterator
def install_split_artifacts(dn):
'''Create the .meta files for a split system
Given a list of artifacts to split, writes new .meta files to
the baserock dir in dn['install'] and copies the files from the
sandbox to the dn['install']
'''
all_splits = []
for i in app.defs.defaults.get_split_rules('stratum'):
all_splits += [i['artifact']]
for index, content in enumerate(dn['contents']):
for stratum, artifacts in content.items():
if artifacts == []:
if config.get('default-splits', []) != []:
for split in config.get('default-splits'):
artifacts += [app.defs.get(stratum)['name'] + split]
else:
for split in all_splits:
artifacts += [os.path.basename(stratum) + split]
dn['contents'][index] = {stratum: artifacts}
for content in dn['contents']:
key = content.keys()[0]
stratum = app.defs.get(key)
move_required_files(dn, stratum, content[key])
def init_logging(flask_app, config):
"""
Set up logging.
Flask automatically writes to stderr in debug mode, so we only configure
the Flask log in production mode.
"""
log_string_format = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
log_date_format = '%Y-%m-%d %H:%M:%S'
log_formatter = logging.Formatter(log_string_format, log_date_format)
try:
log_level = getattr(logging, config.get('logging', 'log_level').upper())
except AttributeError:
raise ValueError("Invalid log level: %s" % log_level)
if not flask_app.debug:
log_file = config.get('logging', 'log_file')
log_handler = logging.FileHandler(log_file)
log_handler.setLevel(log_level)
log_handler.setFormatter(log_formatter)
flask_app.logger.addHandler(log_handler)
if flask_app.debug_db:
# SQL Alchemy logging is very verbose and is only turned on when
# explicitly asked for.
db_log_handler = logging.StreamHandler(sys.stderr)
db_log_handler.setFormatter(log_formatter)
db_logger = logging.getLogger('sqlalchemy.engine')
db_logger.setLevel(logging.INFO)
db_logger.addHandler(db_log_handler)
def init_logging(flask_app, config):
"""
Set up logging.
Flask automatically writes to stderr in debug mode, so we only configure
the Flask log in production mode.
"""
log_string_format = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
log_date_format = '%Y-%m-%d %H:%M:%S'
log_formatter = logging.Formatter(log_string_format, log_date_format)
try:
log_level = getattr(logging,
config.get('logging', 'log_level').upper())
except AttributeError:
raise ValueError("Invalid log level: %s" % log_level)
if not flask_app.debug:
log_file = config.get('logging', 'log_file')
log_handler = logging.FileHandler(log_file)
log_handler.setLevel(log_level)
log_handler.setFormatter(log_formatter)
flask_app.logger.addHandler(log_handler)
if flask_app.debug_db:
# SQL Alchemy logging is very verbose and is only turned on when
# explicitly asked for.
db_log_handler = logging.StreamHandler(sys.stderr)
db_log_handler.setFormatter(log_formatter)
db_logger = logging.getLogger('sqlalchemy.engine')
db_logger.setLevel(logging.INFO)
db_logger.addHandler(db_log_handler)
def post(self):
args = password_reset_post.parse_args()
email = args['email']
user = user_model.get_by_email(email)
if user is None:
api.abort(404, 'User not found')
token = user.get_session_token(expiration=21600, password_reset=True)
if mail.is_email_valid(config.get('mail_sender')):
sender = config.get('mail_sender')
else:
app_id = app_identity.get_application_id()
sender = "%s <noreply@%s.appspotmail.com>" % (app_id, app_id)
try:
message = mail.EmailMessage()
message.sender = sender
message.subject = 'Password reset request'
message.to = email
message.body = '''
You're receiving this email because you requested a password reset.
Visit the following page to reset your password:
%s?reset_token=%s
''' % (config.get('password_reset_url'), token)
message.send()
except:
api.abort(500)
return '', 200
def send_rocketchat_notification(text: str, exc_info: Exception) -> dict:
""" Sends message with specified text to configured Rocketchat channel.
We don't want this method to raise any exceptions, as we don't want to
unintentionally break any kind of error management flow. (We only use
rocket chat notification when something goes wrong).
If you want to know if this method worked or not, you'll have to inspect
the response.
"""
full_message = f"{datetime.now(tz=timezone.utc).isoformat()}\n{text}\n\
{config.get('HOSTNAME')}: {exc_info}\n\
{traceback.format_exception(etype=type(exc_info),value=exc_info,tb=exc_info.__traceback__)}"
result = None
try:
response = requests.post(
config.get('ROCKET_URL_POST_MESSAGE'),
headers={
'X-Auth-Token': config.get('ROCKET_AUTH_TOKEN'),
'X-User-Id': config.get('ROCKET_USER_ID'),
'Content-Type': 'application/json'
},
json={
'channel': config.get('ROCKET_CHANNEL'),
'text': full_message
}
)
result = response.json()
except Exception as exception: # pylint: disable=broad-except
# not doing exc_info=exception - as this causes a lot of noise, and we're more interested
# in the main code!
logger.error('failed to send rocket chat notification %s', exception)
return result
def install_split_artifacts(dn):
'''Create the .meta files for a split system
Given a list of artifacts to split, writes new .meta files to
the baserock dir in dn['install'] and copies the files from the
sandbox to the dn['install']
'''
all_splits = []
for i in app.defs.defaults.get_split_rules('stratum'):
all_splits += [i['artifact']]
for index, content in enumerate(dn['contents']):
for stratum, artifacts in content.items():
if artifacts == []:
if config.get('default-splits', []) != []:
for split in config.get('default-splits'):
artifacts += [app.defs.get(stratum)['name'] + split]
else:
for split in all_splits:
artifacts += [os.path.basename(stratum) + split]
dn['contents'][index] = {stratum: artifacts}
for content in dn['contents']:
key = content.keys()[0]
stratum = app.defs.get(key)
move_required_files(dn, stratum, content[key])
def retrieve_metadata(bibcode, search_identifiers=False):
"""
From the API retrieve the set of metadata we want to know about the record.
"""
params = {
"q": search_identifiers and 'identifier:"{0}"'.format(bibcode) or 'bibcode:"{0}"'.format(bibcode),
"fl": "author,bibcode,identifier",
}
r = requests.get(
config.get("API_SOLR_QUERY_ENDPOINT"),
params=params,
headers={"Accept": "application/json", "Authorization": "Bearer:%s" % config.get("API_TOKEN")},
)
if r.status_code != 200:
raise Exception("{}\n{}\n{}".format(r.status_code, params, r.text))
else:
data = r.json().get("response", {})
if data.get("numFound") == 1:
docs = data.get("docs", [])
return docs[0]
elif data.get("numFound") == 0:
if search_identifiers:
bibcode_cache.setdefault(bibcode, {}) # insert to prevent failed retrievals
raise IgnorableException(u"No metadata found for identifier:{0}".format(bibcode))
else:
return retrieve_metadata(bibcode, search_identifiers=True)
else:
if data.get("numFound") > 10:
raise IgnorableException(u"Insane num of results for {0} ({1})".format(bibcode, data.get("numFound")))
docs = data.get("docs", [])
for d in docs:
for ir in d.get("identifier", []):
if ir.lower().strip() == bibcode.lower().strip():
return d
raise IgnorableException(u"More than one document found for {0}".format(bibcode))
def retrieve_metadata(bibcode, search_identifiers=False):
"""
From the API retrieve the set of metadata we want to know about the record.
"""
params={
'q': search_identifiers and 'identifier:"{0}"'.format(bibcode) or 'bibcode:"{0}"'.format(bibcode),
'fl': 'author,bibcode,identifier'
}
r = requests.get(config.get('API_SOLR_QUERY_ENDPOINT'),
params=params,
headers={'Accept': 'application/json', 'Authorization': 'Bearer:%s' % config.get('API_TOKEN')})
if r.status_code != 200:
raise Exception('{}\n{}\n{}'.format(r.status_code, params, r.text))
else:
data = r.json().get('response', {})
if data.get('numFound') == 1:
docs = data.get('docs', [])
return docs[0]
elif data.get('numFound') == 0:
if search_identifiers:
bibcode_cache.setdefault(bibcode, {}) # insert to prevent failed retrievals
raise IgnorableException(u'No metadata found for identifier:{0}'.format(bibcode))
else:
return retrieve_metadata(bibcode, search_identifiers=True)
else:
if data.get('numFound') > 10:
raise IgnorableException(u'Insane num of results for {0} ({1})'.format(bibcode, data.get('numFound')))
docs = data.get('docs', [])
for d in docs:
for ir in d.get('identifier', []):
if ir.lower().strip() == bibcode.lower().strip():
return d
raise IgnorableException(u'More than one document found for {0}'.format(bibcode))
def relpath(path):
"""
Prefix relative URLs with the Dash app name in case that
the app is deployed on dev environments.
"""
if config.get('base_path'):
return '/{}{}'.format(config.get('base_path'), path)
return path
def plain(dataset=None):
dns = config.get('Instance')['PublicDNS']
port = config.get('Instance')['ShinyPort']
protocol = config.get('Instance')['Protocol']
address = '{0}://{1}:{2}/{3}/'.format(protocol, dns, port, dataset)
return render_template('plain_page.html', dataset=dataset,
analysis_address=address)
def __init__(self):
self.host = config.get("default", "FTP_HOST")
self.user = config.get("default", "FTP_USER")
passw = config.get("default", "FTP_PASSWORD")
logging.info("setting up FTPClient for {}@{}".format(
self.user, self.host))
try:
self.client = FTP(self.host, timeout=10)
self.client.login(user=self.user, passwd=passw)
except Exception as e:
logging.error(e)
def validate_schema(self, schemas, data):
if schemas == {} or \
config.get('schema-validation', False) is False:
return
try:
jsonschema.validate(data, schemas[data.get('kind', None)])
except jsonschema.exceptions.ValidationError as e:
if config.get('schema-validation') == 'strict':
exit(data, 'ERROR: schema validation failed:\n', e)
log(data, 'WARNING: schema validation failed:')
print e
def post(self):
req_data = self.post_parser.parse_args()
user = User.query.filter_by(email=req_data.get("email")).first()
if not user:
return "Wrong email or password!", 404
try:
token = user.encode_user_token(user.id,
config.get("TOKEN_EXPIRES"),
config.get("JWT_SECRET"))
return {"token": token.decode()}, 200
except Exception as e:
return "", 404
def _fix_keys(self, dn, base=None):
'''Normalizes keys for a definition dict and its contents
Some definitions have a 'morph' field which is a relative path. Others
only have a 'name' field, which has no directory part. A few do not
have a 'name'
This sets our key to be 'path', and fixes any missed 'name' to be
the same as 'path' but replacing '/' by '-'
'''
exit = (config.get('check-definitions') == 'exit')
if dn.get('morph'):
if not os.path.isfile(dn.get('morph')):
log('DEFINITION', 'WARNING: missing', dn['morph'], exit=exit)
dn['path'] = self._demorph(dn.pop('morph'))
if 'path' not in dn:
if 'name' not in dn:
log(dn, 'No path, no name?', exit=True)
if config.get('artifact-version', 0) in range(0, 4):
dn['path'] = dn['name']
else:
dn['path'] = os.path.join(self._demorph(base), dn['name'])
if os.path.isfile(dn['path'] + '.morph'):
# morph file exists, but is not mentioned in stratum
# so we ignore it
log(dn,
'WARNING: ignoring',
dn['path'] + '.morph',
exit=exit)
dn['path'] += '.default'
dn['path'] = self._demorph(dn['path'])
dn.setdefault('name', os.path.basename(dn['path']))
if dn.get('name') == config['target']:
config['target'] = dn['path']
n = self._demorph(os.path.basename(dn['name']))
p = self._demorph(os.path.basename(dn['path']))
if os.path.splitext(p)[0] not in n:
log('MORPHS',
'WARNING: %s wrong name' % dn['path'],
dn['name'],
exit=exit)
for system in (dn.get('systems', []) + dn.get('subsystems', [])):
self._fix_keys(system)
def file_done(file):
"""
will be called whenever a Process is finished
:param file: the File object of the File that is done
"""
# delete from "processes"
ProcessRepository.processes.pop(file.id)
# remove original file from disk if desired
if config.getboolean("encoding", "delete_old_file"):
os.remove(file.filename)
# rename file if desired
if config.getboolean("encoding", "rename_enabled"):
rename_search = config.get("encoding", "rename_search")
rename_replace = config.get("encoding", "rename_replace")
# get pathinfo
pathinfo = os.path.split(file.filename)
path = pathinfo[0]
old_filename = pathinfo[1]
# only rename if match occurs
if re.match(rename_search, old_filename):
new_filename = re.sub(rename_search, rename_replace,
old_filename)
# rename output_filename (created by ffmpeg, see process.py) to new_filename
os.rename(path + os.sep + file.output_filename,
path + os.sep + new_filename)
# update status to "finished"
db.session.query(File).filter_by(id=file.id).update(
dict(status=StatusMap.finished.value))
db.session.commit()
# check if it's necessary to start new processes
ProcessRepository.check_and_start_processes()
# notify client
socketio.emit(
"file_done", {
"data": {
"id": file.id,
"count_active": ProcessRepository.count_processes_active(),
"count_queued": ProcessRepository.count_processes_queued(),
"count_total": ProcessRepository.count_processes_total(),
}
})
app.logger.debug("Done with encoding of %s" % file.filename)
def prepare_fetch_hourlies_query(raw_station: dict, start_timestamp: datetime,
end_timestamp: datetime):
""" Prepare url and params to fetch hourly readings from the WFWX Fireweather API.
"""
base_url = config.get('WFWX_BASE_URL')
logger.debug('requesting historic data from %s to %s', start_timestamp,
end_timestamp)
# Prepare query params and query:
query_start_timestamp = math.floor(start_timestamp.timestamp() * 1000)
query_end_timestamp = math.floor(end_timestamp.timestamp() * 1000)
station_id = raw_station['id']
params = {
'startTimestamp': query_start_timestamp,
'endTimestamp': query_end_timestamp,
'stationId': station_id
}
endpoint = (
'/v1/hourlies/search/'
'findHourliesByWeatherTimestampBetweenAndStationIdEqualsOrderByWeatherTimestampAsc'
)
url = f'{base_url}{endpoint}'
return url, params
def install_dependencies(dn, dependencies=None):
'''Install recursed dependencies of dn into dn's sandbox.'''
if dependencies is None:
dependencies = dn.get('build-depends', [])
log(dn, 'Installing dependencies\n', dependencies, verbose=True)
shuffle(dependencies)
for it in dependencies:
dependency = app.defs.get(it)
if os.path.exists(
os.path.join(dn['sandbox'], 'baserock',
dependency['name'] + '.meta')):
# dependency has already been installed
log(dn, 'Already did', dependency['name'], verbose=True)
continue
install_dependencies(dn, dependency.get('build-depends', []))
if (it in dn['build-depends']) or \
(dependency.get('build-mode', 'staging') ==
dn.get('build-mode', 'staging')):
compose(dependency)
if dependency.get('contents'):
install_dependencies(dn, dependency['contents'])
sandbox.install(dn, dependency)
if config.get('log-verbose'):
sandbox.list_files(dn)
async def fetch_detailed_geojson_stations(
session: ClientSession, headers: dict, query_builder: BuildQuery
) -> Tuple[Dict[int, GeoJsonDetailedWeatherStation], Dict[str, int]]:
""" Fetch and marshall geojson station data"""
stations = {}
id_to_code_map = {}
# 1 week seems a reasonable period to cache stations for.
redis_station_cache_expiry: Final = int(
config.get('REDIS_STATION_CACHE_EXPIRY', 604800))
# Put the stations in a nice dictionary.
async for raw_station in fetch_paged_response_generator(
session, headers, query_builder, 'stations', True,
redis_station_cache_expiry):
station_code = raw_station.get('stationCode')
station_status = raw_station.get('stationStatus', {}).get('id')
# Because we can't filter on status in the RSQL, we have to manually exclude stations that are
# not active.
if is_station_valid(raw_station):
id_to_code_map[raw_station.get('id')] = station_code
geojson_station = GeoJsonDetailedWeatherStation(
properties=DetailedWeatherStationProperties(
code=station_code, name=raw_station.get('displayLabel')),
geometry=WeatherStationGeometry(coordinates=[
raw_station.get('longitude'),
raw_station.get('latitude')
]))
stations[station_code] = geojson_station
else:
logger.debug('station %s, status %s', station_code, station_status)
return stations, id_to_code_map
def write_metadata(dn):
if dn.get('kind', 'chunk') == 'chunk':
write_chunk_metafile(dn)
elif dn.get('kind', 'chunk') == 'stratum':
write_stratum_metafiles(dn)
if config.get('check-overlaps', 'ignore') != 'ignore':
check_overlaps(dn)
async def get_stations_by_codes(
station_codes: List[int]) -> List[WeatherStation]:
""" Get a list of stations by code, from WFWX Fireweather API. """
logger.info('Using WFWX to retrieve stations by code')
with EcodivisionSeasons(','.join([str(code) for code in station_codes
])) as eco_division:
async with ClientSession() as session:
header = await get_auth_header(session)
stations = []
# 1 week seems a reasonable period to cache stations for.
redis_station_cache_expiry: Final = int(
config.get('REDIS_STATION_CACHE_EXPIRY', 604800))
# Iterate through "raw" station data.
iterator = fetch_paged_response_generator(
session,
header,
BuildQueryByStationCode(station_codes),
'stations',
use_cache=True,
cache_expiry_seconds=redis_station_cache_expiry)
async for raw_station in iterator:
# If the station is valid, add it to our list of stations.
if is_station_valid(raw_station):
stations.append(parse_station(raw_station, eco_division))
logger.debug('total stations: %d', len(stations))
return stations
def get_current_user():
try:
if not config.get('DEV_SERVER'):
return oauth.get_current_user(SCOPES)
except oauth.Error as e:
pass
return users.get_current_user()
def is_current_user_admin():
try:
if not config.get('DEV_SERVER'):
return oauth.is_current_user_admin(SCOPES)
except oauth.Error as e:
pass
return users.is_current_user_admin()
def get_client_id():
try:
if not config.get('DEV_SERVER'):
return oauth.get_client_id(SCOPES)
except oauth.Error as e:
pass
return None
def install_dependencies(dn, dependencies=None):
'''Install recursed dependencies of dn into dn's sandbox.'''
if dependencies is None:
dependencies = dn.get('build-depends', [])
log(dn, 'Installing dependencies\n', dependencies, verbose=True)
shuffle(dependencies)
for it in dependencies:
dependency = app.defs.get(it)
if os.path.exists(os.path.join(dn['sandbox'], 'baserock',
dependency['name'] + '.meta')):
# dependency has already been installed
log(dn, 'Already did', dependency['name'], verbose=True)
continue
install_dependencies(dn, dependency.get('build-depends', []))
if (it in dn['build-depends']) or \
(dependency.get('build-mode', 'staging') ==
dn.get('build-mode', 'staging')):
compose(dependency)
if dependency.get('contents'):
install_dependencies(dn, dependency['contents'])
sandbox.install(dn, dependency)
if config.get('log-verbose'):
sandbox.list_files(dn)
def CreateAvatar(user):
image_w = 500
image_h = 500
color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
img = Image.new('RGB', (image_w, image_h), color=color)
draw = ImageDraw.Draw(img)
font = ImageFont.truetype(config.get('ROOT_PATH')+'/static/fonts/Roboto-Bold.ttf', size=300)
user_letters = user['first_name'][0]+user['last_name'][0]
text_w, text_h = draw.textsize(user_letters, font=font)
draw.text(
(
(image_w - text_w)/2,
(image_h - text_h - 80)/2,
),
user_letters,
font=font,
fill=(int(color[0] - (color[0]/10)), int(color[1] - (color[1]/10)), int(color[2] - (color[2]/10))),
)
image_path = user['id']+'/avatar.png'
img.save(config['UPLOAD_FOLDER_PROFILE']+image_path)
webp.cwebp(config['UPLOAD_FOLDER_PROFILE']+image_path, config['UPLOAD_FOLDER_PROFILE']+user['id']+'/avatar.webp', "-q 90")
return image_path
def run_build(dn):
''' This is where we run ./configure, make, make install (for example).
By the time we get here, all dependencies for component have already
been assembled.
'''
if config.get('mode', 'normal') == 'no-build':
log(dn, 'SKIPPING BUILD: artifact will be empty')
return
if dn.get('build-mode') != 'bootstrap':
sandbox.ldconfig(dn)
if dn.get('repo'):
repos.checkout(dn)
dn['SOURCE_DATE_EPOCH'] = repos.source_date_epoch(dn['checkout'])
get_build_commands(dn)
env_vars = sandbox.env_vars_for_build(dn)
log(dn, 'Logging build commands to %s' % dn['log'])
for build_step in app.defs.defaults.build_steps:
if dn.get(build_step):
log(dn, 'Running', build_step)
for command in dn.get(build_step, []):
command = 'false' if command is False else command
command = 'true' if command is True else command
sandbox.run_sandboxed(dn, command, env=env_vars,
allow_parallel=('build' in build_step))
if dn.get('devices'):
sandbox.create_devices(dn)
with open(dn['log'], "a") as logfile:
time_elapsed = elapsed(dn['start-time'])
logfile.write('Elapsed_time: %s\n' % time_elapsed)
def install_contents(dn, contents=None):
''' Install contents (recursively) into dn['sandbox'] '''
if contents is None:
contents = dn.get('contents', [])
log(dn, 'Installing contents\n', contents, verbose=True)
shuffle(contents)
for it in contents:
item = app.defs.get(it)
if os.path.exists(os.path.join(dn['sandbox'],
'baserock', item['name'] + '.meta')):
# content has already been installed
log(dn, 'Already installed', item['name'], verbose=True)
continue
for i in item.get('contents', []):
install_contents(dn, [i])
if item.get('build-mode', 'staging') != 'bootstrap':
if not get_cache(item):
compose(item)
sandbox.install(dn, item)
if config.get('log-verbose'):
log(dn, 'Added contents\n', contents)
sandbox.list_files(dn)
def parse_files(self, directory):
schemas = self.load_schemas()
with chdir(directory):
for dirname, dirnames, filenames in os.walk('.'):
filenames.sort()
dirnames.sort()
if '.git' in dirnames:
dirnames.remove('.git')
for filename in filenames:
if filename.endswith(('.def', '.morph')):
path = os.path.join(dirname, filename)
data = self._load(path)
if data is not None:
self.validate_schema(schemas, data)
data['path'] = path[2:]
self._fix_keys(data)
self._tidy_and_insert_recursively(data)
if config.get('mode') == 'parse-only':
with open(config['result-file'], 'w') as f:
f.write(
yaml.dump(self._data,
default_flow_style=False,
Dumper=ExplicitDumper))
log('RESULT', 'Parsed definitions data in yaml format is at',
config['result-file'])
os._exit(0)
def get_ads_orcid_profile(orcidid):
r = requests.get(config.get('API_ORCID_EXPORT_PROFILE') % orcidid,
headers={'Accept': 'application/json', 'Authorization': 'Bearer:%s' % config.get('API_TOKEN')})
if r.status_code != 200:
return None
else:
return r.json()
def install_contents(dn, contents=None):
''' Install contents (recursively) into dn['sandbox'] '''
if contents is None:
contents = dn.get('contents', [])
log(dn, 'Installing contents\n', contents, verbose=True)
shuffle(contents)
for it in contents:
item = app.defs.get(it)
if os.path.exists(
os.path.join(dn['sandbox'], 'baserock',
item['name'] + '.meta')):
# content has already been installed
log(dn, 'Already installed', item['name'], verbose=True)
continue
for i in item.get('contents', []):
install_contents(dn, [i])
if item.get('build-mode', 'staging') != 'bootstrap':
if not get_cache(item):
compose(item)
sandbox.install(dn, item)
if config.get('log-verbose'):
log(dn, 'Added contents\n', contents)
sandbox.list_files(dn)
def layout(params):
# This function must return a layout for the homepage
return dbc.Container(
[
html.Div(
[
html.H5('Choose a page to view'),
],
className='px-3 py-3 pt-md-5 pb-md-4 mx-auto text-center'
),
dbc.CardDeck(
[
dbc.Card(
[
dbc.CardHeader(page.get('name', '')),
dbc.CardBody(
[
html.P(page.get('description', '')),
html.A('View', href=page.get('path', '/'), className='btn btn-block btn-outline-primary')
],
className='d-flex flex-column'
)
],
className='mb-3 mx-auto'
)
for page in config.get('pages', [])
],
className='homepage mb-3'
)
],
)
def get_public_orcid_profile(orcidid):
r = requests.get(config.get('API_ORCID_PROFILE_ENDPOINT') % orcidid,
headers={'Accept': 'application/json'})
if r.status_code != 200:
return None
else:
return r.json()
def write_metadata(defs, component):
if component.get('kind', 'chunk') == 'chunk':
write_chunk_metafile(defs, component)
elif component.get('kind', 'chunk') == 'stratum':
write_stratum_metafiles(defs, component)
if config.get('check-overlaps', 'ignore') != 'ignore':
check_overlaps(defs, component)
def run(self):
"""
run the encoding
"""
# probe file first
frame_count = self.ffmpeg_probe_frame_count()
if frame_count == -1:
app.logger.debug("Probing of " + self.file.filename +
" failed - aborting...")
ProcessRepository.file_failed(self.file)
return
# app.logger.debug("Probing of " + file.filename + " successful.. frame count: " + str(frame_count))
split_path = os.path.split(self.file.filename)
path = split_path[0]
original_filename = split_path[1]
filename_noext = os.path.split(
os.path.splitext(original_filename)[0])[1]
# form output filename and store it in self.file for later use
self.file.output_filename = filename_noext + ".pyencode"
cmd = ["ffmpeg"]
cmd.extend(["-i", self.file.filename])
# add parameters from config
cmd.extend(shlex.split(config.get("encoding", "parameters")))
cmd.extend(["-y", path + os.sep + self.file.output_filename])
app.logger.debug("Starting encoding of " + self.file.filename +
" with %s" % " ".join(cmd))
for info in self.run_ffmpeg(cmd, frame_count):
if info["return_code"] != -1:
app.logger.debug(
"Error occured while running ffmpeg. Last lines of output: "
)
app.logger.debug("\n".join(info["last_lines"]))
ProcessRepository.file_failed(self.file)
return
# store information in database
# convert kB to bytes
info["ffmpeg_size"] *= 1024
# we don't need the return_code anymore (and don't want to store it)
info.pop("return_code")
# update file in DB
File.query.filter_by(id=self.file.id).update(info)
db.session.commit()
# update self.file
for k in info:
setattr(self.file, k, info[k])
# tell ProcessRepository there's some progress going on
ProcessRepository.file_progress(self.file)
if self.active:
ProcessRepository.file_done(self.file)
def file_done(file):
"""
will be called whenever a Process is finished
:param file: the File object of the File that is done
"""
# delete from "processes"
ProcessRepository.processes.pop(file.id)
# remove original file from disk if desired
if config.getboolean("encoding", "delete_old_file"):
os.remove(file.filename)
# rename file if desired
if config.getboolean("encoding", "rename_enabled"):
rename_search = config.get("encoding", "rename_search")
rename_replace = config.get("encoding", "rename_replace")
# get pathinfo
pathinfo = os.path.split(file.filename)
path = pathinfo[0]
old_filename = pathinfo[1]
# only rename if match occurs
if re.match(rename_search, old_filename):
new_filename = re.sub(rename_search, rename_replace, old_filename)
# rename output_filename (created by ffmpeg, see process.py) to new_filename
os.rename(path + os.sep + file.output_filename, path + os.sep + new_filename)
# update status to "finished"
db.session.query(File).filter_by(id=file.id).update(dict(status=StatusMap.finished.value))
db.session.commit()
# check if it's necessary to start new processes
ProcessRepository.check_and_start_processes()
# notify client
socketio.emit("file_done", {
"data": {
"id": file.id,
"count_active": ProcessRepository.count_processes_active(),
"count_queued": ProcessRepository.count_processes_queued(),
"count_total": ProcessRepository.count_processes_total(),
}
})
app.logger.debug("Done with encoding of %s" % file.filename)
def _fix_keys(self, dn, base=None):
'''Normalizes keys for a definition dict and its contents
Some definitions have a 'morph' field which is a relative path. Others
only have a 'name' field, which has no directory part. A few do not
have a 'name'
This sets our key to be 'path', and fixes any missed 'name' to be
the same as 'path' but replacing '/' by '-'
'''
exit = (config.get('check-definitions') == 'exit')
if dn.get('morph'):
if not os.path.isfile(dn.get('morph')):
log('DEFINITION', 'WARNING: missing', dn['morph'], exit=exit)
dn['path'] = self._demorph(dn.pop('morph'))
if 'path' not in dn:
if 'name' not in dn:
log(dn, 'No path, no name?', exit=True)
if config.get('artifact-version') in range(0, 4):
dn['path'] = dn['name']
else:
dn['path'] = os.path.join(self._demorph(base), dn['name'])
if os.path.isfile(dn['path'] + '.morph'):
# morph file exists, but is not mentioned in stratum
# so we ignore it
log(dn, 'WARNING: ignoring', dn['path'] + '.morph',
exit=exit)
dn['path'] += '.default'
dn['path'] = self._demorph(dn['path'])
dn.setdefault('name', os.path.basename(dn['path']))
if dn.get('name') == config['target']:
config['target'] = dn['path']
n = self._demorph(os.path.basename(dn['name']))
p = self._demorph(os.path.basename(dn['path']))
if os.path.splitext(p)[0] not in n:
log('MORPHS', 'WARNING: %s wrong name' % dn['path'], dn['name'],
exit=exit)
for system in (dn.get('systems', []) + dn.get('subsystems', [])):
self._fix_keys(system)
def _tidy_and_insert_recursively(self, dn):
'''Insert a definition and its contents into the dictionary.
Takes a dict containing the content of a definition file.
Inserts the definitions referenced or defined in the
'build-depends' and 'contents' keys of `definition` into the
dictionary, and then inserts `definition` itself into the
dictionary.
'''
# handle morph syntax oddities...
for index, component in enumerate(dn.get('build-depends', [])):
self._fix_keys(component)
dn['build-depends'][index] = self._insert(component)
# The 'contents' field in the internal data model corresponds to the
# 'chunks' field in a stratum .morph file, or the 'strata' field in a
# system .morph file.
dn['contents'] = dn.get('contents', [])
if type(dn.get('chunks', [])) is not list:
log('DEFINITIONS', 'WARNING: %s chunks must be list:' % dn['path'],
dn.get('chunks', []), exit=True)
if type(dn.get('strata', [])) is not list:
log('DEFINITIONS', 'WARNING: %s strata must be list:' % dn['path'],
dn.get('strata', []), exit=True)
dn['contents'] += dn.pop('chunks', []) + dn.pop('strata', [])
lookup = {}
for index, component in enumerate(dn['contents']):
self._fix_keys(component, dn['path'])
lookup[component['name']] = component['path']
if component['name'] == dn['name']:
log(dn, 'WARNING: %s contains' % dn['path'], dn['name'])
for x, it in enumerate(component.get('build-depends', [])):
if it not in lookup:
# it is defined as a build depend, but hasn't actually been
# defined yet...
dependency = {'name': it}
self._fix_keys(dependency, dn['path'])
lookup[it] = dependency['path']
component['build-depends'][x] = lookup[it]
component['build-depends'] = (dn.get('build-depends', []) +
component.get('build-depends', []))
if config.get('artifact-version', 0) not in [0, 1, 2, 3, 4, 5]:
c = self._data.get(component['path'])
if c and 'build-depends' in c:
component['build-depends'] += c['build-depends']
splits = component.get('artifacts', [])
dn['contents'][index] = {self._insert(component): splits}
return self._insert(dn)
def prepare_fetch_dailies_for_all_stations_query(time_of_interest: datetime,
page_count: int):
""" Prepare url and params for fetching dailies(that's forecast and observations for noon) for all.
stations. """
base_url = config.get('WFWX_BASE_URL')
noon_date = _get_noon_date(time_of_interest)
timestamp = int(noon_date.timestamp() * 1000)
# one could filter on recordType.id==FORECAST or recordType.id==ACTUAL but we want it all.
params = {
'query': f'weatherTimestamp=={timestamp}',
'page': page_count,
'size': config.get('WFWX_MAX_PAGE_SIZE', 1000)
}
endpoint = ('/v1/dailies/rsql')
url = f'{base_url}{endpoint}'
logger.info('%s %s', url, params)
return url, params
async def get_client() -> Generator[Tuple[AioBaseClient, str], None, None]:
""" Return AioBaseClient client and bucket
"""
server = config.get('OBJECT_STORE_SERVER')
user_id = config.get('OBJECT_STORE_USER_ID')
secret_key = config.get('OBJECT_STORE_SECRET')
bucket = config.get('OBJECT_STORE_BUCKET')
session = get_session()
async with session.create_client('s3',
endpoint_url=f'https://{server}',
aws_secret_access_key=secret_key,
aws_access_key_id=user_id) as client:
try:
yield client, bucket
finally:
del client
def test_get(app: Flask) -> None:
with app.app_context(): # type: ignore
app.config = Mock()
app.config.get = Mock(return_value='value') # type: ignore
assert get('key') == 'value'
app.config.get.assert_called_once_with('key') # type: ignore
def packagemanager():
#是否已同意许可证协议
license_config = {
'True': packagemanager_already_accept,
'False': pacekagemanager_new
}
#返回view
return license_config[config.get("License", "License")](request)
def get(obj):
if obj.get('template'):
template = obj['template']
else:
template = config.get()['defaults']['template']
name = obj['name']
return render_template(template + '''.html''', name=name)
def compose(dn):
'''Work through defs tree, building and assembling until target exists'''
if type(dn) is not dict:
dn = app.defs.get(dn)
# if we can't calculate cache key, we can't create this component
if cache_key(dn) is False:
if 'tried' not in dn:
log(dn, 'No cache_key, so skipping compose')
dn['tried'] = True
return False
# if dn is already cached, we're done
if get_cache(dn):
return cache_key(dn)
log(dn, "Composing", dn['name'], verbose=True)
# if we have a kbas, look there to see if this component exists
if config.get('kbas-url') and not config.get('reproduce'):
with claim(dn):
if get_remote(dn):
config['counter'].increment()
return cache_key(dn)
# we only work with user-specified arch
if 'arch' in dn and dn['arch'] != config['arch']:
return None
# Create composite components (strata, systems, clusters)
systems = dn.get('systems', [])
shuffle(systems)
for system in systems:
for s in system.get('subsystems', []):
subsystem = app.defs.get(s['path'])
compose(subsystem)
compose(system['path'])
with sandbox.setup(dn):
install_contents(dn)
build(dn) # bring in 'build-depends', and run make
return cache_key(dn)
def compose(dn):
'''Work through defs tree, building and assembling until target exists'''
if type(dn) is not dict:
dn = app.defs.get(dn)
# if we can't calculate cache key, we can't create this component
if cache_key(dn) is False:
if 'tried' not in dn:
log(dn, 'No cache_key, so skipping compose')
dn['tried'] = True
return False
# if dn is already cached, we're done
if get_cache(dn):
return cache_key(dn)
log(dn, "Composing", dn['name'], verbose=True)
# if we have a kbas, look there to see if this component exists
if config.get('kbas-url') and not config.get('reproduce'):
with claim(dn):
if get_remote(dn):
config['counter'].increment()
return cache_key(dn)
# we only work with user-specified arch
if 'arch' in dn and dn['arch'] != config['arch']:
return None
# Create composite components (strata, systems, clusters)
systems = dn.get('systems', [])
shuffle(systems)
for system in systems:
for s in system.get('subsystems', []):
subsystem = app.defs.get(s['path'])
compose(subsystem)
compose(system['path'])
with sandbox.setup(dn):
install_contents(dn)
build(dn) # bring in 'build-depends', and run make
return cache_key(dn)
def run(self):
"""
run the encoding
"""
# probe file first
frame_count = self.ffmpeg_probe_frame_count()
if frame_count == -1:
app.logger.debug("Probing of " + self.file.filename + " failed - aborting...")
ProcessRepository.file_failed(self.file)
return
# app.logger.debug("Probing of " + file.filename + " successful.. frame count: " + str(frame_count))
split_path = os.path.split(self.file.filename)
path = split_path[0]
original_filename = split_path[1]
filename_noext = os.path.split(os.path.splitext(original_filename)[0])[1]
# form output filename and store it in self.file for later use
self.file.output_filename = filename_noext + ".pyencode"
cmd = ["ffmpeg"]
cmd.extend(["-i", self.file.filename])
# add parameters from config
cmd.extend(shlex.split(config.get("encoding", "parameters")))
cmd.extend(["-y", path + os.sep + self.file.output_filename])
app.logger.debug("Starting encoding of " + self.file.filename + " with %s" % " ".join(cmd))
for info in self.run_ffmpeg(cmd, frame_count):
if info["return_code"] != -1:
app.logger.debug("Error occured while running ffmpeg. Last lines of output: ")
app.logger.debug("\n".join(info["last_lines"]))
ProcessRepository.file_failed(self.file)
return
# store information in database
# convert kB to bytes
info["ffmpeg_size"] *= 1024
# we don't need the return_code anymore (and don't want to store it)
info.pop("return_code")
# update file in DB
File.query.filter_by(id=self.file.id).update(info)
db.session.commit()
# update self.file
for k in info:
setattr(self.file, k, info[k])
# tell ProcessRepository there's some progress going on
ProcessRepository.file_progress(self.file)
if self.active:
ProcessRepository.file_done(self.file)
def verify_session_token(cls, token):
if token is None:
return False, None
s = Serializer(config.get('secret_key'))
try:
data = s.loads(token)
except SignatureExpired:
return False, None
except BadSignature:
return False, None
return True, data
def write_metafile(rules, splits, dn):
metadata = {'products': [{'artifact': a,
'components': sorted(set(splits[a]))}
for a, r in rules]}
if dn.get('kind', 'chunk') == 'chunk':
metadata['repo'] = dn.get('repo')
metadata['ref'] = dn.get('ref')
else:
if config.get('artifact-version', 0) not in range(0, 2):
metadata['repo'] = config['defdir']
metadata['ref'] = config['def-version']
if config.get('artifact-version', 0) not in range(0, 1):
metadata['cache'] = dn.get('cache')
meta = os.path.join(dn['baserockdir'], dn['name'] + '.meta')
with open(meta, "w") as f:
yaml.safe_dump(metadata, f, default_flow_style=False)
def install_contents(defs, component, contents=None):
''' Install contents (recursively) into component['sandbox'] '''
component = defs.get(component)
if contents is None:
contents = component.get('contents', [])
log(component, 'Installing contents\n', contents, verbose=True)
shuffle(contents)
for it in contents:
this = defs.get(it)
if os.path.exists(os.path.join(component['sandbox'],
'baserock', this['name'] + '.meta')):
# content has already been installed
log(component, 'Already installed', this['name'], verbose=True)
continue
if component.get('kind', 'chunk') == 'system':
artifacts = []
for content in component['contents']:
if content.keys()[0] == this['path']:
artifacts = content[this['path']]
break
if artifacts != [] or config.get('default-splits', []) != []:
compose(defs, this)
install_split_artifacts(defs, component, this, artifacts)
continue
for i in this.get('contents', []):
install_contents(defs, component, [i])
if this.get('build-mode', 'staging') != 'bootstrap':
if not get_cache(defs, this):
compose(defs, this)
sandbox.install(defs, component, this)
if config.get('log-verbose'):
log(component, 'Added contents\n', contents)
sandbox.list_files(component)
def _insert(self, new_def):
'''Insert a new definition into the dictionary, return the key.
Takes a dict representing a single definition.
If a definition with the same 'path' doesn't exist, just add
`new_def` to the dictionary.
If a definition with the same 'path' already exists, extend the
existing definition with the contents of `new_def` unless it
and the new definition both contain a 'ref'. If any keys are
duplicated in the existing definition, output a warning.
If `new_def` contains a sha: field (which needs to be 40 chars),
this overrides ref: field
'''
exit = (config.get('check-definitions') == 'exit')
dn = self._data.get(new_def['path'])
if dn:
if (dn.get('ref') is None or new_def.get('ref') is None):
for key in new_def:
if key is not 'name':
dn[key] = new_def[key]
# If a sha was specified, we want to build it instead of the ref
# but preserve the ref in the output <target>.yml file.
if dn.get('sha'):
if len(dn['sha']) != 40:
log(new_def, 'ERROR: invalid sha:', dn['sha'], exit=True)
dn['orig_ref'] = dn['ref']
dn['ref'] = dn['sha']
if dn['name'] != new_def['name']:
log(new_def, 'WARNING: %s also named as' % new_def['name'],
dn['name'], exit=exit)
dn['name'] = new_def['name']
for key in new_def:
if dn.get(key) and new_def[key] and dn[key] != new_def[key]:
log(new_def,
'WARNING: multiple definitions of %s \n' % key,
'%s | %s' % (dn.get(key), new_def[key]), exit=exit)
else:
self._data[new_def['path']] = new_def
return new_def['path']
def log_changes(dn, tmpdir, old_defs, ref):
do_git_log = False
old_def = old_defs.get(dn['path'])
log_file = os.path.join(tmpdir, dn['name'])
with open(log_file, 'w') as f:
keys = set(dn) - set(['tree', 'cache'])
for key in keys:
try:
old_value = old_def.get(key)
except:
old_value = None
if dn[key] != old_value:
f.write('[%s] Value changed: %s\n' % (dn['path'], key))
if type(dn[key]) is str:
f.write('%s | %s\n' % (old_value, dn[key]))
if type(dn[key]) is not str and type(dn[key]) is not float:
if old_value:
for x in old_value:
f.write(repr(x))
f.write('\n vvv\n')
if dn[key]:
for x in dn[key]:
f.write(repr(x))
f.write('\n\n')
if dn.get('kind', 'chunk') == 'chunk' and config.get('release-cmd'):
log(dn, 'Logging git change history', tmpdir)
try:
gitdir = os.path.join(config['gits'],
get_repo_name(dn['repo']))
if not os.path.exists(gitdir):
mirror(dn['name'], dn['repo'])
elif not mirror_has_ref(gitdir, ref):
update_mirror(dn['name'], dn['repo'], gitdir)
with chdir(gitdir):
text = dn['ref'] + '..'
if old_def and old_def.get('ref'):
text += old_def['ref']
f.write(check_output(config['release-command'] + [text]))
except:
log(dn, 'WARNING: Failed to log git changes')
if os.stat(log_file).st_size == 0:
os.remove(log_file)
def write_stratum_metafiles(stratum):
'''Write the .meta files for a stratum to the baserock dir
The split rules are used to divide up the installed components into
artifacts in the 'products' list in the stratum .meta file. Each artifact
contains a list of chunk artifacts which match the stratum splitting rules
'''
log(stratum['name'], 'Splitting', stratum.get('kind'))
rules, splits = compile_rules(stratum)
for item in stratum['contents']:
chunk = app.defs.get(item)
if chunk.get('build-mode', 'staging') == 'bootstrap':
continue
metadata = get_metadata(chunk)
split_metadata = {'ref': metadata.get('ref'),
'repo': metadata.get('repo'),
'products': []}
if config.get('artifact-version', 0) not in range(0, 1):
split_metadata['cache'] = metadata.get('cache')
chunk_artifacts = app.defs.get(chunk).get('artifacts', {})
for artifact, target in chunk_artifacts.items():
splits[target].append(artifact)
for product in metadata['products']:
for artifact, rule in rules:
if rule.match(product['artifact']):
split_metadata['products'].append(product)
splits[artifact].append(product['artifact'])
break
meta = os.path.join(stratum['baserockdir'], chunk['name'] + '.meta')
with open(meta, "w") as f:
yaml.safe_dump(split_metadata, f, default_flow_style=False)
write_metafile(rules, splits, stratum)
def check_overlaps(dn):
if set(config['new-overlaps']) <= set(config['overlaps']):
config['new-overlaps'] = []
return
overlaps_found = False
config['new-overlaps'] = list(set(config['new-overlaps']))
for path in config['new-overlaps']:
log(dn, 'WARNING: overlapping path', path)
for filename in os.listdir(dn['baserockdir']):
with open(os.path.join(dn['baserockdir'], filename)) as f:
for line in f:
if path[1:] in line:
log(filename, 'WARNING: overlap at', path[1:])
overlaps_found = True
break
if config.get('check-overlaps') == 'exit':
log(dn, 'Overlaps found', config['new-overlaps'], exit=True)
config['overlaps'] = list(set(config['new-overlaps'] + config['overlaps']))
config['new-overlaps'] = []
def get_build_commands(dn):
'''Get commands specified in d, plus commands implied by build-system
The containing definition may point to another definition file (using
the 'path' field in YBD's internal data model) that contains build
instructions, or it may only specify a predefined build system, using
'build-system' field.
The definition containing build instructions can specify a predefined
build-system and then override some or all of the command sequences it
defines.
If the definition file doesn't exist and no build-system is specified,
this function will scan the contents the checked-out source repo and try
to autodetect what build system is used.
'''
if dn.get('kind', None) == "system":
# Systems must run their integration scripts as install commands
dn['install-commands'] = gather_integration_commands(dn)
return
exit = True if config.get('check-definitions') == 'exit' else False
if 'build-system' in dn:
bs = dn['build-system']
log(dn, 'Defined build system is', bs)
else:
files = os.listdir(dn['checkout'])
bs = app.defs.defaults.detect_build_system(files)
if bs == 'manual' and 'install-commands' not in dn:
if dn.get('kind', 'chunk') == 'chunk':
print dn
log(dn, 'WARNING: No install-commands, manual build-system',
exit=exit)
log(dn, 'WARNING: Assumed build system is', bs)
for build_step in app.defs.defaults.build_steps:
if dn.get(build_step, None) is None:
commands = app.defs.defaults.build_systems[bs].get(build_step, [])
dn[build_step] = commands
def parse_article(article, share):
if article.is_parsed():
return notify_new_article(article, share)
endpoint = config['ARTICLE_PARSE_ENDPOINT']
payload = {
'token': config.get('DIFFBOT_TOKEN', None),
'url': article.url
}
resp = requests.get(endpoint, params=payload)
if not resp.ok:
app.logger.error('Failed to parse %s (error code: %s). Reason: %s'
% (article, resp.status_code, resp.json() if resp.json() else 'unknown'))
return
json = resp.json()
if 'resolved_url' in json:
article.url = json['resolved_url']
elif 'url' in json:
article.url = json['url']
else:
app.logger.warning("Parse article did not return any url for %s (%s)" % (article, json))
article.icon = json.get('icon')
article.title = (json['title'][:252] + '...') if len(json['title']) > 255 else json['title']
article.text = json['text']
article.date = json['date']
article.author = json.get('author', '')
article.parse_date = datetime.utcnow()
share.parsed = True
article.save()
share.save()
notify_new_article(article, share)
def _insert(self, new_def):
'''Insert a new definition into the dictionary, return the key.
Takes a dict representing a single definition.
If a definition with the same 'path' doesn't exist, just add
`new_def` to the dictionary.
If a definition with the same 'path' already exists, extend the
existing definition with the contents of `new_def` unless it
and the new definition both contain a 'ref'. If any keys are
duplicated in the existing definition, output a warning.
'''
exit = (config.get('check-definitions') == 'exit')
dn = self._data.get(new_def['path'])
if dn:
if (dn.get('ref') is None or new_def.get('ref') is None):
for key in new_def:
if key is not 'name':
dn[key] = new_def[key]
if dn['name'] != new_def['name']:
log(new_def, 'WARNING: %s also named as' % new_def['name'],
dn['name'], exit=exit)
dn['name'] = new_def['name']
for key in new_def:
if dn.get(key) and new_def[key] and dn[key] != new_def[key]:
log(new_def,
'WARNING: multiple definitions of %s \n' % key,
'%s | %s' % (dn.get(key), new_def[key]), exit=exit)
else:
self._data[new_def['path']] = new_def
return new_def['path']
def _run(self, args, config):
""" Main entry point. """
try:
solr_url = config.get("solr", "url").rstrip("/") + "/"
solr = scorched.SolrInterface(solr_url)
except:
raise cli.CliError("Unable to connect to solr: %s" % solr_url)
if args.action in ("add", "add-all"):
database_config = dict(config.items("database"))
db = app.database.get_engine(database_config)
if args.action == "add":
self.add_models(db, solr, args.models.split(","))
else:
self.add_models(db, solr)
solr.optimize()
self._logger.info("Added requested documents and optimized index.")
elif args.action in ("delete", "delete-all"):
if args.action == "delete":
self.delete_models(solr, args.models.split(","))
else:
solr.delete_all()
solr.optimize()
self._logger.info("Deleted requested documents and optimized index.")
elif args.action == "optimize":
solr.optimize()
self._logger.info("Optimized index.")
elif args.action == "schema":
schema_url = urljoin(solr_url, "schema")
self.schema(schema_url)
def move_required_files(dn, stratum, artifacts):
log(dn, 'Installing %s artifacts' % stratum['name'], artifacts)
stratum_metadata = get_metadata(stratum)
split_stratum_metadata = {}
split_stratum_metadata['products'] = []
to_keep = []
for product in stratum_metadata['products']:
for artifact in artifacts:
if artifact == product['artifact']:
to_keep += product['components']
split_stratum_metadata['products'].append(product)
log(dn, 'Splitting artifacts:', artifacts, verbose=True)
log(dn, 'Splitting components:', to_keep, verbose=True)
baserockpath = os.path.join(dn['install'], 'baserock')
if not os.path.isdir(baserockpath):
os.mkdir(baserockpath)
split_stratum_metafile = os.path.join(baserockpath,
stratum['name'] + '.meta')
with open(split_stratum_metafile, "w") as f:
yaml.safe_dump(split_stratum_metadata, f, default_flow_style=False)
for path in stratum['contents']:
chunk = app.defs.get(path)
if chunk.get('build-mode', 'staging') == 'bootstrap':
continue
try:
metafile = path_to_metafile(chunk)
with open(metafile, "r") as f:
filelist = []
metadata = yaml.safe_load(f)
split_metadata = {'ref': metadata.get('ref'),
'repo': metadata.get('repo'),
'products': []}
if config.get('artifact-version', 0) not in range(0, 1):
metadata['cache'] = dn.get('cache')
for product in metadata['products']:
if product['artifact'] in to_keep:
filelist += product.get('components', [])
# handle old artifacts still containing 'files'
filelist += product.get('files', [])
split_metadata['products'].append(product)
if split_metadata['products'] != []:
split_metafile = os.path.join(baserockpath,
os.path.basename(metafile))
with open(split_metafile, "w") as f:
yaml.safe_dump(split_metadata, f,
default_flow_style=False)
log(dn, 'Splits split_metadata is\n', split_metadata,
verbose=True)
log(dn, 'Splits filelist is\n', filelist, verbose=True)
copy_file_list(dn['sandbox'], dn['install'], filelist)
except:
import traceback
traceback.print_exc()
log(dn, 'Failed to install split components', exit=True)
#! /usr/bin/python
# -*- coding:utf-8 -*-
from app import app
from app import config
if __name__ == '__main__':
app.run(host=config.get('flask', 'url'),
port=config.getint('flask', 'port'),
debug=True)
''' Performs randomized requests
Will flip coins or roll dice as requested and configured.
'''
from app import bot, config
from random import choice
COIN_OPTIONS = config.get("coin_sides", ["heads", "tails"])
DIE = {type: map(str, range(1, sides+1)) for (type, sides)
in config.get("die", {"dice": 6}).items()}
@bot.listen("@me: flip coin")
def flip_coin(channel, user):
''' flips a coin for the user
Will flip a coin for the user
usage: `@me flip coin`
'''
result = choice(COIN_OPTIONS)
channel.post("{!s}: flipped a {!s}".format(user, result))
return result
@bot.listen("@me: roll <({}):die_type>"
.format("|".join(DIE.keys())), regex=True)
@bot.listen("@me: roll <int:number> <({}):die_type>"
.format("|".join(DIE.keys())), regex=True)
def roll_dice(channel, user, die_type, number=1):
''' rolls a variable number of various die types for the user
Will roll one or many die of the requested type.
