def __init__(self,
dependencies,
tiles_fname,
features_dir,
output_tile_fname,
wait_time=None,
conf_fname=None,
callback=None):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.features_dir = '"{0}"'.format(features_dir)
self.output_file = '-o "{0}"'.format(output_tile_fname)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
if wait_time is None:
self.wait_time = ''
else:
self.wait_time = '-w {0}'.format(wait_time)
self.callback = callback
self.dependencies = dependencies
self.memory = 400
self.time = 20
self.output = output_tile_fname
def __init__(self, ConfigFile, Children=[]):
# Initialize Job specific stuff in base class
# -------------------------------------------
Job.__init__(self, ConfigFile)
self.Children = Children
def __init__(self,
matches_work_dir,
tiles_fname,
temp_files_prefix,
threads_num=1,
wait_time=None,
conf_fname=None):
Job.__init__(self)
self.already_done = False
self.dependencies = []
self.features_fnames1_list = []
self.features_fnames2_list = []
self.output_files_list = []
self.index_pairs_list = []
self.tiles_fname = '"{0}"'.format(tiles_fname)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
if wait_time is None:
self.wait_time = ''
else:
self.wait_time = '-w {0}'.format(wait_time)
self.memory = 450
self.time = 100
self.threads = threads_num
self.threads_str = "-t {0}".format(threads_num)
self.matches_work_dir = matches_work_dir
self.temp_files_prefix = temp_files_prefix
def __init__(self,
dependencies,
tiles_fname,
features_fname1,
features_fname2,
corr_output_file,
index_pair,
wait_time=None,
conf_fname=None):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.features_fname1 = '"{0}"'.format(features_fname1)
self.features_fname2 = '"{0}"'.format(features_fname2)
self.index_pair = ':'.join(index_pair)
self.output_file = '-o "{0}"'.format(corr_output_file)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
if wait_time is None:
self.wait_time = ''
else:
self.wait_time = '-w {0}'.format(wait_time)
self.dependencies = dependencies
self.memory = 400
self.time = 20
self.output = corr_output_file
def __init__(self,
dependencies,
tiles_fname,
output_dir,
layer,
scale,
from_x,
from_y,
to_x,
to_y,
jar_file,
output_file,
threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.output_dir = '-o "{0}"'.format(output_dir)
self.jar_file = '-j "{0}"'.format(jar_file)
# Make the from-layer and to-layer the same layer
self.from_layer = '--from_layer {0}'.format(layer)
self.to_layer = '--to_layer {0}'.format(layer)
self.scale = '--scale {0}'.format(scale)
self.from_x = '--from_x {0}'.format(from_x)
self.from_y = '--from_y {0}'.format(from_y)
self.to_x = '--to_x {0}'.format(to_x)
self.to_y = '--to_y {0}'.format(to_y)
self.dependencies = dependencies
self.threads = threads_num
self.threads_str = "-t {0}".format(threads_num)
self.memory = 33000
self.time = 300
self.is_java_job = True
self.output = output_file
def __init__(self,
dependencies,
tiles_fname,
matches_list_file,
opt_output_file,
conf_fname=None,
threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.matches_list_file = '"{0}"'.format(matches_list_file)
self.output_file = '-o "{0}"'.format(opt_output_file)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
# if fixed_tiles is None:
# self.fixed_tiles = ''
# else:
# self.fixed_tiles = '-f {0}'.format(" ".join(str(f) for f in fixed_tiles))
self.dependencies = dependencies
# self.threads = threads_num
# self.threads_str = "-t {0}".format(threads_num)
self.memory = 4000
self.time = 600
self.output = opt_output_file
def __init__(self, dependencies, outputs, tiles_fnames, corr_fnames, model_fnames, fixed_layers, output_dir, max_layer_distance, jar_file, conf_fname=None, skip_layers=None, threads_num=1, manual_matches=None):
Job.__init__(self)
self.already_done = False
self.tiles_fnames = '--tile_files {0}'.format(" ".join(tiles_fnames))
self.corr_fnames = '--corr_files {0}'.format(" ".join(corr_fnames))
self.model_fnames = '--model_files {0}'.format(" ".join(model_fnames))
self.output_dir = '-o "{0}"'.format(output_dir)
if fixed_layers is None:
self.fixed_layers = ''
else:
self.fixed_layers = '-f {0}'.format(" ".join(str(f) for f in fixed_layers))
self.jar_file = '-j "{0}"'.format(jar_file)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
if skip_layers is None:
self.skip_layers = ''
else:
self.skip_layers = '-s "{0}"'.format(skip_layers)
if manual_matches is None:
self.manual_matches = ''
else:
self.manual_matches = '-M {0}'.format(" ".join(manual_matches))
self.max_layer_distance = '-d {}'.format(max_layer_distance)
self.threads = threads_num
self.threads_str = '-t {0}'.format(threads_num)
self.dependencies = dependencies
self.memory = 50000
self.time = 1400
self.is_java_job = True
self.output = outputs
def __init__(self,doc):
Job.__init__(self, doc)
try:
self.format = self.format
except AttributeError:
self.format = "json"
try:
self.coll_type = self.coll_type
except AttributeError:
self.coll_type = None
self._dict_values = {}
self._dict_values["sources"] = {
"filename": "%s/export_%s_sources_%s.%s" %(self.project_name, self.name, self.date, self.format),
"format": self.format,
"fields": 'url,origin,date.date',
}
self._dict_values["logs"] = {
"filename": "%s/export_%s_logs_%s.%s" %(self.project_name,self.name, self.date, self.format),
"format":self.format,
"fields": 'url,code,scope,status,msg',
}
self._dict_values["results"] = {
"filename": "%s/export_%s_results_%s.%s" %(self.project_name,self.name, self.date, self.format),
"format":self.format,
"fields": 'url,domain,title,content.content,outlinks.url,crawl_date',
}
def __init__(self, ssh_client, script_path):
Job.__init__(self)
self.script_path = script_path
self.ssh_client = ssh_client
self._raw_output = []
self._last_log = None
self._parser = RemoteScriptOutputParser()
def __init__(self, dependencies, outputs, input_dir, output_dir):
Job.__init__(self)
#### TODO - stopped here!
self.already_done = False
self.input_dir = '{0}'.format(input_dir)
self.output_dir = '-o "{0}"'.format(output_dir)
self.dependencies = dependencies
self.memory = 2000
self.time = 400
self.output = outputs
def __init__(self, tiles_fname, output_dir, output_file, jar_file):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.output_dir = '-o "{0}"'.format(output_dir)
self.jar_file = '-j "{0}"'.format(jar_file)
self.dependencies = []
self.memory = 3000
self.time = 10
self.is_java_job = True
self.output = output_file
def __init__(self, dependencies, tiles_fnames, output_dir, jar_file,
outputs):
Job.__init__(self)
self.already_done = False
self.tiles_fnames = '{0}'.format(" ".join(tiles_fnames))
self.output_dir = '-o "{0}"'.format(output_dir)
self.jar_file = '-j "{0}"'.format(jar_file)
self.dependencies = dependencies
self.memory = 4000
self.time = 100
self.is_java_job = True
self.output = outputs
def __init__(self, tiles_fname, output_dir, jar_file, threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.output_dir = '-o "{0}"'.format(output_dir)
self.jar_file = '-j "{0}"'.format(jar_file)
self.dependencies = [ ]
self.threads = threads_num
self.threads_str = "-t {0}".format(threads_num)
self.memory = 64000
self.time = 200
self.is_java_job = True
self.output = output_dir
def __init__(self, dependencies, in_dir, open_sea_dragon, processes_num=1):
Job.__init__(self)
self.already_done = False
self.in_dir = '"{0}"'.format(in_dir)
self.open_sea_dragon = ''
if open_sea_dragon is True:
self.open_sea_dragon = '-s'
self.dependencies = dependencies
self.threads = processes_num
self.processes_str = '-p {0}'.format(processes_num)
self.memory = 16000
self.time = 500
self.is_java_job = False
self.output = os.path.join(in_dir, "1")
def __init__(self, dependencies, tiles_fname, corr_fname, output_fname, jar_file, conf_fname=None):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.corr_fname = '"{0}"'.format(corr_fname)
self.output_file = '-o "{0}"'.format(output_fname)
self.jar_file = '-j "{0}"'.format(jar_file)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
self.dependencies = dependencies
self.memory = 5000
self.time = 30
self.is_java_job = True
self.output = output_fname
def __init__(self, tiles_fname1, features_dir1, tiles_fname2, features_dir2, output_fname, conf_fname=None, threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fname1 = '"{0}"'.format(tiles_fname1)
self.features_dir1 = '"{0}"'.format(features_dir1)
self.tiles_fname2 = '"{0}"'.format(tiles_fname2)
self.features_dir2 = '"{0}"'.format(features_dir2)
self.output_fname = '-o "{0}"'.format(output_fname)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
self.dependencies = [ ]
#self.threads = threads_num
#self.threads_str = "-t {0}".format(threads_num)
self.memory = 1000
self.time = 300
self.output = output_fname
def __init__(self, dependencies, tiles_fname1, features_fname1, tiles_fname2, features_fname2, corr_output_file, jar_file, conf_fname=None, threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fname1 = '"{0}"'.format(tiles_fname1)
self.features_fname1 = '"{0}"'.format(features_fname1)
self.tiles_fname2 = '"{0}"'.format(tiles_fname2)
self.features_fname2 = '"{0}"'.format(features_fname2)
self.output_file = '-o "{0}"'.format(corr_output_file)
self.jar_file = '-j "{0}"'.format(jar_file)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
self.dependencies = dependencies
self.threads = threads_num
self.threads_str = "-t {0}".format(threads_num)
self.memory = 5000
self.time = 30
self.is_java_job = True
self.output = corr_output_file
def __init__(self,
dependencies,
tiles_fname,
out_fname,
width,
jar_file,
threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.output_file = '-o "{0}"'.format(out_fname)
self.jar_file = '-j "{0}"'.format(jar_file)
self.width = '-w {0}'.format(width)
self.threads = threads_num
self.threads_str = '-t {0}'.format(threads_num)
self.dependencies = dependencies
self.memory = 34000
self.time = 500
self.is_java_job = True
self.output = out_fname
def __init__(self, dependencies, tiles_fname, output_file, jar_file, meshes_dir=None, conf_fname=None, threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.output_file = '-o "{0}"'.format(output_file)
self.jar_file = '-j "{0}"'.format(jar_file)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
if meshes_dir is None:
self.meshes_dir = ''
else:
self.meshes_dir = '--meshes_dir "{0}"'.format(meshes_dir)
self.dependencies = dependencies
self.threads = threads_num
self.threads_str = "-t {0}".format(threads_num)
self.memory = 13000
self.time = 400
self.is_java_job = True
self.output = output_file
def __init__(self, dependencies, outputs, tiles_fnames, corr_fnames, output_dir, max_layer_distance, conf_fname=None, skip_layers=None, threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fnames = '--tile_files {0}'.format(" ".join(tiles_fnames))
self.corr_fnames = '--corr_files {0}'.format(" ".join(corr_fnames))
self.output_dir = '-o "{0}"'.format(output_dir)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
if skip_layers is None:
self.skip_layers = ''
else:
self.skip_layers = '-s "{0}"'.format(skip_layers)
self.max_layer_distance = '-d {}'.format(max_layer_distance)
self.threads = threads_num
self.threads_str = '-t {0}'.format(threads_num)
self.dependencies = dependencies
self.memory = 10000
self.time = 800
self.output = outputs
def __init__(self,
image_file,
output_dir,
output_file,
tile_size,
output_pattern,
output_type,
processes_num=1):
Job.__init__(self)
self.already_done = False
self.image_file = '"{0}"'.format(image_file)
self.output_dir = '-o "{0}"'.format(output_dir)
self.output_type = '--output_type "{0}"'.format(output_type)
self.output_pattern = '--output_pattern "{0}"'.format(output_pattern)
self.tile_size = '--tile_size "{0}"'.format(tile_size)
self.dependencies = []
self.threads = processes_num
self.processes_str = "--processes_num {0}".format(processes_num)
self.memory = 32000
self.time = 300
self.output = output_file
def __init__(self,
tiles_fname,
output_file,
tile_index,
conf_fname=None,
threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.tile_index = '{0}'.format(tile_index)
self.output_file = '-o "{0}"'.format(output_file)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
self.dependencies = []
# self.threads = threads_num
# self.threads_str = "-t {0}".format(threads_num)
self.memory = 4000
self.time = 20
self.output = output_file
def __init__(self,
tiles_fname,
sifts_work_dir,
temp_output_list_file,
conf_fname=None,
threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.tile_indices_list = []
self.output_files_list = []
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
self.dependencies = []
# self.threads = threads_num
# self.threads_str = "-t {0}".format(threads_num)
self.memory = 2800
self.time = 100
self.sifts_work_dir = sifts_work_dir
self.temp_output_list_file = temp_output_list_file
def __init__(self, dependencies, tiles_fname1, tiles_fname2, pre_match_fname, output_fname, targeted_mfov, conf_fname=None, threads_num=1, auto_add_model=False):
Job.__init__(self)
self.already_done = False
self.tiles_fname1 = '"{0}"'.format(tiles_fname1)
self.tiles_fname2 = '"{0}"'.format(tiles_fname2)
self.pre_match_fname = '"{0}"'.format(pre_match_fname)
self.targeted_mfov = '{0}'.format(targeted_mfov)
self.output_fname = '-o "{0}"'.format(output_fname)
if conf_fname is None:
self.conf_fname = ''
else:
self.conf_fname = '-c "{0}"'.format(conf_fname)
#if auto_add_model:
# self.auto_add_model = '--auto_add_model'
#else:
# self.auto_add_model = ''
self.threads = threads_num
self.threads_str = '-t {0}'.format(threads_num)
self.dependencies = dependencies
self.memory = 10000
self.time = 800
self.output = output_fname
def __init__(self, dependencies, tiles_fname, out_dir, tile_size, output_type, jar_file, output_pattern=None, blend_type=None, threads_num=1):
Job.__init__(self)
self.already_done = False
self.tiles_fname = '"{0}"'.format(tiles_fname)
self.output_dir = '-o "{0}"'.format(out_dir)
self.jar_file = '-j "{0}"'.format(jar_file)
self.tile_size = '-s {0}'.format(tile_size)
self.threads = threads_num
self.threads_str = '-t {0}'.format(threads_num)
if blend_type is None:
self.blend_type = ''
else:
self.blend_type = '-b {0}'.format(blend_type)
self.output_type = '--output_type "{0}"'.format(output_type)
if output_pattern is None:
self.output_pattern = ''
else:
self.output_pattern = '--output_pattern "{0}"'.format(output_pattern)
self.dependencies = dependencies
self.memory = 35000
self.time = 600
self.is_java_job = True
self.output = out_dir
def __init__(self, command, dispatcher):
self.occurrences = 0
Job.__init__(self, command, dispatcher)
def __init__(self, agent_obj):
Job.__init__(self, agent_obj)
self.conn = None
self.key = None
def __init__(self, agent_obj, c_job):
Job.__init__(self, agent_obj)
self.conn = None
self.key = None
self.connect_job = c_job
def __init__(self, job, sitedef, scheme, next_start=None, start_offset=None):
Job.__init__(self, job, scheme)
self.id = job[u'_id'] if job else None
self.sitedef = sitedef
self.next_start = next_start
self.start_offset = start_offset
