def processSpecification(specification,
extensionRules,
projects_directory,
completeFile=None,
outputGraph=False,
threads=1,
loglevel=None,
global_variables='',
initializers=None,
skipValidation=False,
passthrus=[]):
"""
Perform a plugin operation on all projects in directory
:param projects_directory: directory of projects
:param extensionRules: rules that determine which nodes are extended
:param specification: the specification file (see batch project)
:return: None
"""
from functools import partial
batch = loadJSONGraph(specification)
rules = parseRules(extensionRules)
global_state = updateAndInitializeGlobalState(dict(), global_variables, initializers)
global_state['skipvalidation'] = skipValidation
if len(passthrus) > 0:
global_state['passthrus'] = passthrus
if loglevel is not None:
logging.getLogger('maskgen').setLevel(logging.INFO if loglevel is None else int(loglevel))
set_logging_level(logging.INFO if loglevel is None else int(loglevel))
if batch is None:
return
iterator = pick_projects(projects_directory)
iterator.extend(pick_zipped_projects(projects_directory))
processor = BatchProcessor(completeFile, iterator, threads=threads)
func = partial(processAnyProject, batch, rules, outputGraph, '.', global_state)
return processor.process(func)
def processZippedProject(batchSpecification, extensionRules, project, workdir=None, global_state=dict()):
import tempfile
import shutil
dir = tempfile.mkdtemp()
try:
extract_archive(project, dir)
for project in pick_projects(dir):
sm = _processProject(batchSpecification, extensionRules, project,
workdir=workdir,
global_state=global_state)
sm.export(workdir)
finally:
shutil.rmtree(dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-u', '--username', help="optional username", required=False)
parser.add_argument('--projects', help='Directory of projects')
args = parser.parse_args()
initialize(maskGenPreferences, username=args.username)
project_list = pick_projects(args.projects)
with open(os.path.join(args.projects,'ErrorReport_' + str(os.getpid()) + '.csv'), 'wb') as csvfile:
error_writer = csv.writer(csvfile, delimiter = ' ', quotechar='|', quoting=csv.QUOTE_MINIMAL)
for project in project_list:
try:
validate_export(error_writer, project, scenario_model.loadProject(project))
except Exception as e:
print (project)
print (str(e))
def main(argv=sys.argv[1:]):
from functools import partial
parser = argparse.ArgumentParser()
parser.add_argument('--threads',
default=1,
required=False,
help='number of projects to build')
parser.add_argument('-d', '--projects', help='directory of projects')
parser.add_argument('--completeFile',
default=None,
help='A file recording completed projects')
args = parser.parse_args(argv)
iterator = pick_projects(args.projects)
processor = BatchProcessor(args.completeFile,
iterator,
threads=args.threads)
func = partial(fix_projects, args)
return processor.process(func)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--projects', help='Directory of projects')
parser.add_argument('-cf',
'--completefile',
required=True,
help='Projects to Completed')
parser.add_argument('-t',
'--threads',
required=False,
default=1,
help='Threads')
args = parser.parse_args()
plugins.loadPlugins()
project_list = pick_projects(args.projects)
processor = BatchProcessor(args.completefile,
project_list,
threads=int(args.threads))
processor.process(rerunexif)
def processProject(directory, file_to_process):
import thread
"""
:param args:
:param functions:
:param file_to_process:
:return:
@type file_to_process : str
"""
dir = os.path.join(tempfile.gettempdir(),
'probe_{}_{}'.format(os.getpid(), thread.get_ident()))
os.mkdir(dir)
try:
fetchfromS3(dir, 'medifor/par/journal/ingested', file_to_process)
extract_archive(os.path.join(dir, file_to_process), dir)
for project in pick_projects(dir):
archive_probes(project, directory=directory)
finally:
shutil.rmtree(dir)
return []
def processProject(args, file_to_process):
"""
:param args:
:param functions:
:param file_to_process:
:return:
@type file_to_process : str
"""
if not file_to_process.endswith('tgz') and os.path.exists(
os.path.join(args.tempfolder, file_to_process)):
dir = os.path.join(args.tempfolder, file_to_process)
fetch = False
else:
dir = tempfile.mkdtemp(
dir=args.tempfolder) if args.tempfolder else tempfile.mkdtemp()
fetch = True
try:
file_to_process = (file_to_process + '.tgz')
if fetch:
fetchfromS3(dir, args.downloadfolder, file_to_process)
extract_archive(os.path.join(dir, file_to_process), dir)
logging.getLogger('maskgen').info(
file_to_process +
' downloaded and extracted, proceeding to update')
for project in pick_projects(dir):
log = []
if args.autoupdate:
log = run_auto_update(project)
print(log)
if args.functions is not None:
log = perform_update(project, args)
print(log)
#export
return log
finally:
if fetch:
try:
shutil.rmtree(dir)
except OSError:
pass
def processProject(file_to_process):
import thread
"""
:param args:
:param functions:
:param file_to_process:
:return:
@type file_to_process : str
"""
dir = os.path.join(tempfile.gettempdir(),
'repull_{}_{}'.format(os.getpid(), thread.get_ident()))
os.mkdir(dir)
try:
fetchfromS3(dir, 'medifor/par/journal/ingested',
file_to_process[0] + '.tgz')
extract_archive(os.path.join(dir, file_to_process[0] + '.tgz'), dir)
for project in pick_projects(dir):
perform_update(project, file_to_process[1],
'medifor/par/journal/staging')
finally:
shutil.rmtree(dir)
return []
def processProject(args, file_to_process):
"""
:param args:
:param functions:
:param file_to_process:
:return:
@type file_to_process : str
"""
project_temp = tempfile.mkdtemp(dir=args.tempfolder) if args.tempfolder else tempfile.mkdtemp() #make our working directory for the project.
try:
original_Archive= file_to_process+'.tgz'
if not os.path.exists(os.path.join(args.tempfolder, original_Archive)): #check if the original is at the top of tempdir
fetchfromS3(args.tempfolder, args.downloadfolder, original_Archive)
extract_archive(os.path.join(args.tempfolder, original_Archive), project_temp) #extract into the working tempdir
logging.getLogger('maskgen').info(file_to_process + ' downloaded and extracted, proceeding to update')
for project in pick_projects(project_temp):
log = []
if args.autoupdate:
log = run_auto_update(project)
print('auto update complete.')
print(log)
if args.functions is not None:
log, failed = perform_update(project, args)
print('updates complete')
print(log)
if failed:
logging.getLogger('maskgen').info(
file_to_process + ' PROJECT FAILED ')
return log
finally:
if not args.keep:
try:
shutil.rmtree(project_temp) #Toss the working directory
os.remove(os.path.join(args.tempfolder, file_to_process+'.tgz'))#Toss the original.
except OSError:
pass