def execute_task(sync_job,task_metadata,task_logger):
"""
execute the task, based on tasks_metadata
"""
task_name = taskname(sync_job,task_metadata)
task_status = sync_job['status'].get_task_status(task_metadata[TASK_TYPE_INDEX])
if task_status.is_succeed:
#this task has been executed successfully
return
if sync_job['status'].is_failed:
#some proceding task are failed,so can't execute this task
if task_status.shared:
#this task is shared, but this task can't executed for this job, change the task's status object to a private status object
from slave_sync_status import SlaveSyncTaskStatus
sync_job['status'].set_task_status(task_metadata[TASK_TYPE_INDEX],SlaveSyncTaskStatus())
return
task_logger.info("Begin to process the {3}task ({0} - {1} {2}).".format(task_metadata[TASK_TYPE_INDEX],task_name,sync_job["job_file"],"shared " if task_status.shared else ""))
sync_job['status'].last_process_time = now()
task_status.last_process_time = now()
try:
task_status.del_message("message")
task_metadata[TASK_HANDLER_INDEX](sync_job,task_metadata,task_status)
if not task_status.get_message("message"):
task_status.set_message("message","succeed")
task_status.succeed()
task_logger.info("Succeed to process the {3}task ({0} - {1} {2}).".format(task_metadata[TASK_TYPE_INDEX],task_name,sync_job["job_file"],"shared " if task_status.shared else ""))
except:
task_status.failed()
message = traceback.format_exc()
task_status.set_message("message",message)
task_logger.error("Failed to Process the {4}task ({0} - {1} {2}).{3}".format(task_metadata[TASK_TYPE_INDEX],task_name,sync_job["job_file"],message,"shared " if task_status.shared else ""))
def execute_prepare_task(sync_job, task_metadata, task_logger):
"""
execute the prepare task, based on tasks_metadata
"""
task_name = taskname(sync_job, task_metadata)
task_status = sync_job["status"].get_task_status(task_metadata[TASK_TYPE_INDEX])
task_logger.info(
"Begin to process the task ({0} - {1} {2}).".format(
task_metadata[TASK_TYPE_INDEX], task_name, sync_job["job_file"]
)
)
sync_job["status"].last_process_time = now()
task_status.last_process_time = now()
try:
task_metadata[TASK_HANDLER_INDEX](sync_job, task_metadata, task_status)
if not task_status.get_message("message"):
task_status.set_message("message", "succeed")
task_status.succeed()
task_logger.info(
"Succeed to process the task ({0} - {1} {2}).".format(
task_metadata[TASK_TYPE_INDEX], task_name, sync_job["job_file"]
)
)
except:
task_status.failed()
message = traceback.format_exc()
task_status.set_message("message", message)
task_logger.error(
"Failed to Process the task ({0} - {1} {2}).{3}".format(
task_metadata[TASK_TYPE_INDEX], task_name, sync_job["job_file"], message
)
)
def execute_task(sync_job, task_metadata, task_logger):
"""
execute the task, based on tasks_metadata
"""
task_name = taskname(sync_job, task_metadata)
task_status = sync_job["status"].get_task_status(task_metadata[TASK_TYPE_INDEX])
if task_status.is_succeed:
# this task has been executed successfully
return
if task_status.is_processed:
# this task is already processed, maybe triggered by other tasks
sync_job["status"].execute_succeed = False
return
if not sync_job["status"].execute_succeed:
# some proceding task are failed,so can't execute this task
if task_status.shared:
# this task is shared, but this task can't executed for this job, change the task's status object to a private status object
from slave_sync_status import SlaveSyncTaskStatus
sync_job["status"].set_task_status(task_metadata[TASK_TYPE_INDEX], SlaveSyncTaskStatus())
return
task_logger.info(
"Begin to process the {3}task ({0} - {1} {2}).".format(
task_metadata[TASK_TYPE_INDEX], task_name, sync_job["job_file"], "shared" if task_status.shared else ""
)
)
sync_job["status"].last_process_time = now()
task_status.last_process_time = now()
try:
task_metadata[TASK_HANDLER_INDEX](sync_job, task_metadata, task_status)
if not task_status.get_message("message"):
task_status.set_message("message", "succeed")
task_status.succeed()
task_logger.info(
"Succeed to process the {3}task ({0} - {1} {2}).".format(
task_metadata[TASK_TYPE_INDEX], task_name, sync_job["job_file"], "shared" if task_status.shared else ""
)
)
except:
task_status.failed()
sync_job["status"].execute_succeed = False
message = traceback.format_exc()
task_status.set_message("message", message)
task_logger.error(
"Failed to Process the {4}task ({0} - {1} {2}).{3}".format(
task_metadata[TASK_TYPE_INDEX],
task_name,
sync_job["job_file"],
message,
"shared" if task_status.shared else "",
)
)
def send_last_poll_time(cls):
if feedback_disabled:
return
if not hasattr(cls, "_listen_channels"):
cls._listen_channels = ",".join(LISTEN_CHANNELS)
last_poll_time = now()
sql = """
DO
$$BEGIN
IF EXISTS (SELECT 1 FROM {0}.monitor_slaveserver WHERE name='{1}') THEN
UPDATE {0}.monitor_slaveserver SET listen_channels='{2}', last_poll_time='{3}',code_version='{4}' WHERE name='{1}';
ELSE
INSERT INTO {0}.monitor_slaveserver (name,register_time,listen_channels,last_poll_time,code_version) VALUES ('{1}','{3}','{2}','{3}','{4}');
END IF;
END$$;
""".format(
MASTER_PGSQL_SCHEMA,
SLAVE_NAME,
cls._listen_channels,
last_poll_time,
"{0} ({1})".format(get_version(), CODE_BRANCH),
)
sql_cmd[len(sql_cmd) - 1] = sql
sql_process = subprocess.Popen(sql_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
sql_output = sql_process.communicate()
if sql_output[1] and sql_output[1].strip():
logger.info("stderr: {}".format(sql_output[1]))
if sql_process.returncode != 0:
logger.error(
"Update the last sync info in master db failed with return code ({0})".format(sql_process.returncode)
)
def send_last_sync_time(cls, pull_status):
if feedback_disabled:
return
last_sync_time = now()
last_sync_message = str(pull_status).replace("'", "''")
sql = """
DO
$$BEGIN
UPDATE {0}.monitor_slaveserver SET last_sync_time='{2}', last_sync_message='{3}' WHERE name='{1}';
END$$;
""".format(
MASTER_PGSQL_SCHEMA, SLAVE_NAME, last_sync_time, last_sync_message
)
# logger.info("hg pull status notify: \r\n" + sql)
sql_cmd[len(sql_cmd) - 1] = sql
sql_process = subprocess.Popen(sql_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
sql_output = sql_process.communicate()
if sql_output[1] and sql_output[1].strip():
logger.info("stderr: {}".format(sql_output[1]))
if sql_process.returncode != 0:
logger.error(
"Update the last sync info in master db failed with return code ({0})".format(sql_process.returncode)
)
def stage_succeed(self,stage):
"""
Set a flag indicate this state is processed successfully
"""
if "stages" not in self:
self["stages"] = {}
if stage not in self["stages"]:
self["stages"][stage] = {}
self["stages"][stage]['status'] = True
self["stages"][stage]['last_process_time'] = date_to_str(now())
self._modified = True
def execute_prepare_task(sync_job,task_metadata,task_logger):
"""
execute the prepare task, based on tasks_metadata
"""
task_name = taskname(sync_job,task_metadata)
task_status = sync_job['status'].get_task_status(task_metadata[TASK_TYPE_INDEX])
task_logger.info("Begin to process the task ({0} - {1} {2}).".format(task_metadata[TASK_TYPE_INDEX],task_name,sync_job["job_file"]))
sync_job['status'].last_process_time = now()
task_status.last_process_time = now()
try:
task_metadata[TASK_HANDLER_INDEX](sync_job,task_metadata,task_status)
if not task_status.get_message("message"):
task_status.set_message("message","succeed")
task_status.succeed()
task_logger.info("Succeed to process the task ({0} - {1} {2}).".format(task_metadata[TASK_TYPE_INDEX],task_name,sync_job["job_file"]))
except:
task_status.failed()
message = traceback.format_exc()
task_status.set_message("message",message)
task_logger.error("Failed to Process the task ({0} - {1} {2}).{3}".format(task_metadata[TASK_TYPE_INDEX],task_name,sync_job["job_file"],message))
def stage_succeed(self, stage):
"""
Set a flag indicate this state is processed successfully
"""
if "stages" not in self:
self["stages"] = {}
if stage not in self["stages"]:
self["stages"][stage] = {}
self["stages"][stage]['status'] = True
self["stages"][stage]['last_process_time'] = date_to_str(now())
self._modified = True
def load_metafile(sync_job):
meta_file = sync_job.get('meta',None)
if not meta_file:
#no meta file, all meta datas are embeded into the sync_job
return
task_status = sync_job['status'].get_task_status("load_metadata")
task_status.last_process_time = now()
if task_status.is_succeed:
#this task has been executed successfully,
#load the json file and add the meta data into sync_job
local_meta_file = task_status.get_message("meta_file")
with open(local_meta_file,"r") as f:
meta_data = json.loads(f.read())
sync_job.update(meta_data)
sync_job['meta']['local_file'] = local_meta_file
return
logger.info("Begin to load meta data for job({})".format(sync_job['job_file']))
#download from borg master
temp_file = os.path.join(CACHE_PATH,"job.meta.json")
if sync_job['action'] == "remove":
#for remove action, doesn't check md5 because of the following case.
# 1. Publish an layer to repository
# 2. Slave pull from the repository and then publish the layer, at this time, the md5 of the layer's metadata file is 'A'
# 3. Publish the layer again, and now, the md5 of the layer's metadata file is 'B',
# 4. Remove the layer from repository.
# 5. Slave pull from the repository, only the last version will be fetched, and intermediate versions are ignored. so the publish action in step 3 is invisible for slave cient.
# 6. Slave client try to fetch the meta file from master and compare the md5 , and found: file's md5 is 'B', but md5 data in repository is 'A', doesn't match.
download_file(meta_file["file"],temp_file,task_status,None)
else:
download_file(meta_file["file"],temp_file,task_status,meta_file.get('md5',None))
meta_data = None
with open(temp_file,"r") as f:
meta_data = json.loads(f.read())
sync_job.update(meta_data)
if "workspace" in sync_job:
local_meta_file = os.path.join(CACHE_PATH,"{}.{}.meta.json".format(sync_job["workspace"],sync_job["name"]))
else:
local_meta_file = os.path.join(CACHE_PATH,"{}.meta.json".format(sync_job["name"]))
try:
os.remove(local_meta_file)
except:
#file not exist, ignore
pass
#rename to meta file
os.rename(temp_file,local_meta_file)
sync_job['meta']['local_file'] = local_meta_file
task_status.set_message("message","Succeed to download meta data from master.")
task_status.set_message("meta_file",local_meta_file)
task_status.succeed()
def send_last_sync_time(cls,pull_status):
if feedback_disabled: return
last_sync_time = now()
last_sync_message = str(pull_status).replace("'","''")
sql = """
DO
$$BEGIN
UPDATE {0}.monitor_slaveserver SET last_sync_time='{2}', last_sync_message='{3}' WHERE name='{1}';
END$$;
""".format(MASTER_PGSQL_SCHEMA,SLAVE_NAME,last_sync_time,last_sync_message)
#logger.info("hg pull status notify: \r\n" + sql)
sql_cmd[len(sql_cmd) - 1] = sql
sql_process = subprocess.Popen(sql_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
sql_output = sql_process.communicate()
if sql_output[1] and sql_output[1].strip():
logger.info("stderr: {}".format(sql_output[1]))
if sql_process.returncode != 0:
logger.error("Update the last sync info in master db failed with return code ({0})".format(sql_process.returncode))
def send_last_poll_time(cls):
if feedback_disabled: return
if not hasattr(cls,"_listen_channels"):
cls._listen_channels = ",".join(LISTEN_CHANNELS)
last_poll_time = now()
sql = """
DO
$$BEGIN
IF EXISTS (SELECT 1 FROM {0}.monitor_slaveserver WHERE name='{1}') THEN
UPDATE {0}.monitor_slaveserver SET listen_channels='{2}', last_poll_time='{3}',code_version='{4}' WHERE name='{1}';
ELSE
INSERT INTO {0}.monitor_slaveserver (name,register_time,listen_channels,last_poll_time,code_version) VALUES ('{1}','{3}','{2}','{3}','{4}');
END IF;
END$$;
""".format(MASTER_PGSQL_SCHEMA,SLAVE_NAME,cls._listen_channels,last_poll_time,"{0} ({1})".format(get_version(),CODE_BRANCH))
sql_cmd[len(sql_cmd) - 1] = sql
sql_process = subprocess.Popen(sql_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
sql_output = sql_process.communicate()
if sql_output[1] and sql_output[1].strip():
logger.info("stderr: {}".format(sql_output[1]))
if sql_process.returncode != 0:
logger.error("Update the last sync info in master db failed with return code ({0})".format(sql_process.returncode))
def send_feature_sync_status(cls,task,remove=False):
if feedback_disabled:
logger.info("Notify feature is disabled.")
return
try:
sync_succeed = task["status"].is_succeed
sync_message = str(task["status"]).replace("'","''")
sync_time = task["status"].last_process_time or now()
if sync_succeed:
if remove:
#remove publish succeed.
sql_template = """
DELETE FROM {0}.monitor_publishsyncstatus AS b
USING {0}.monitor_slaveserver AS a
WHERE a.id = b.slave_server_id
AND a.name = '{1}'
AND b.publish = '{2}'
"""
else:
#update publish succeed
sql_template = """
DO
$$BEGIN
IF EXISTS (SELECT 1 FROM {0}.monitor_slaveserver a JOIN {0}.monitor_publishsyncstatus b ON a.id=b.slave_server_id WHERE a.name='{1}' AND b.publish='{2}') THEN
UPDATE {0}.monitor_publishsyncstatus AS b SET deploied_job_id='{3}', deploied_job_batch_id='{4}', deploy_message='{5}',deploy_time= {6}, sync_job_id = null, sync_job_batch_id = null, sync_time = null, sync_message = null,preview_file={7},spatial_type='{8}'
FROM {0}.monitor_slaveserver AS a
WHERE b.slave_server_id = a.id AND a.name='{1}' AND b.publish='{2}';
ELSE
INSERT INTO {0}.monitor_publishsyncstatus
(slave_server_id,publish,deploied_job_id,deploied_job_batch_id,deploy_message,deploy_time,sync_job_id,sync_job_batch_id,sync_message,sync_time,preview_file,spatial_type)
SELECT id,'{2}','{3}','{4}','{5}',{6},null,null,null,null,{7},'{8}'
FROM {0}.monitor_slaveserver
WHERE name = '{1}';
END IF;
END$$;
"""
elif not remove:
sql_template = """
DO
$$BEGIN
IF EXISTS (SELECT 1 FROM {0}.monitor_slaveserver a JOIN {0}.monitor_publishsyncstatus b ON a.id=b.slave_server_id WHERE a.name='{1}' AND b.publish='{2}') THEN
UPDATE {0}.monitor_publishsyncstatus AS b SET sync_job_id='{3}', sync_job_batch_id='{4}', sync_message='{5}',sync_time= {6},spatial_type='{8}'
FROM {0}.monitor_slaveserver AS a
WHERE b.slave_server_id = a.id AND a.name='{1}' AND b.publish='{2}';
ELSE
INSERT INTO {0}.monitor_publishsyncstatus
(slave_server_id,publish,deploied_job_id,deploied_job_batch_id,deploy_message,deploy_time,sync_job_id,sync_job_batch_id,sync_message,sync_time,spatial_type)
SELECT id,'{2}',null,null,null,null,'{3}','{4}','{5}',{6},'{8}'
FROM {0}.monitor_slaveserver
WHERE name = '{1}';
END IF;
END$$;
"""
preview_file = task["status"].get_task_status("get_layer_preview").get_message("preview_file") or None
preview_file = "'{}'".format(preview_file) if preview_file else "null"
sql = sql_template.format(MASTER_PGSQL_SCHEMA, SLAVE_NAME,task['name'], task.get("job_id"), task.get("job_batch_id"), sync_message, "'{0}'".format(sync_time) if sync_time else 'null',preview_file,task.get('spatial_type',''))
#logger.info("Feature sync status notify: \r\n" + sql)
sql_cmd[len(sql_cmd) - 1] = sql
sql_process = subprocess.Popen(sql_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
sql_output = sql_process.communicate()
if sql_output[1] and sql_output[1].strip():
logger.info("stderr: {}".format(sql_output[1]))
if sql_process.returncode != 0 and sql_output[1].find("ERROR") >= 0:
cls._save_failed_sql(sql)
logger.error("Update sync status of task ({0}) in master db failed with return code ({1})".format(task['job_file'],sql_process.returncode))
except:
logger.error("Update sync status of task ({0}) in master db failed. {1}".format(task['job_file'],traceback.format_exc()))
def send_feature_sync_status(cls,task,remove=False):
if feedback_disabled:
logger.info("Notify feature is disabled.")
return
try:
sync_succeed = task["status"].is_succeed
sync_message = str(task["status"]).replace("'","''")
sync_time = task["status"].last_process_time or now()
if sync_succeed:
if remove:
#remove publish succeed.
sql_template = """
DELETE FROM {0}.monitor_publishsyncstatus AS b
USING {0}.monitor_slaveserver AS a
WHERE a.id = b.slave_server_id
AND a.name = '{1}'
AND b.publish = '{2}'
"""
else:
#update publish succeed
sql_template = """
DO
$$BEGIN
IF EXISTS (SELECT 1 FROM {0}.monitor_slaveserver a JOIN {0}.monitor_publishsyncstatus b ON a.id=b.slave_server_id WHERE a.name='{1}' AND b.publish='{2}') THEN
UPDATE {0}.monitor_publishsyncstatus AS b SET deploied_job_id='{3}', deploied_job_batch_id='{4}', deploy_message='{5}',deploy_time= {6}, sync_job_id = null, sync_job_batch_id = null, sync_time = null, sync_message = null,preview_file={7},spatial_type='{8}'
FROM {0}.monitor_slaveserver AS a
WHERE b.slave_server_id = a.id AND a.name='{1}' AND b.publish='{2}';
ELSE
INSERT INTO {0}.monitor_publishsyncstatus
(slave_server_id,publish,deploied_job_id,deploied_job_batch_id,deploy_message,deploy_time,sync_job_id,sync_job_batch_id,sync_message,sync_time,preview_file,spatial_type)
SELECT id,'{2}','{3}','{4}','{5}',{6},null,null,null,null,{7},'{8}'
FROM {0}.monitor_slaveserver
WHERE name = '{1}';
END IF;
END$$;
"""
else:
sql_template = """
DO
$$BEGIN
IF EXISTS (SELECT 1 FROM {0}.monitor_slaveserver a JOIN {0}.monitor_publishsyncstatus b ON a.id=b.slave_server_id WHERE a.name='{1}' AND b.publish='{2}') THEN
UPDATE {0}.monitor_publishsyncstatus AS b SET sync_job_id='{3}', sync_job_batch_id='{4}', sync_message='{5}',sync_time= {6},spatial_type='{8}'
FROM {0}.monitor_slaveserver AS a
WHERE b.slave_server_id = a.id AND a.name='{1}' AND b.publish='{2}';
ELSE
INSERT INTO {0}.monitor_publishsyncstatus
(slave_server_id,publish,deploied_job_id,deploied_job_batch_id,deploy_message,deploy_time,sync_job_id,sync_job_batch_id,sync_message,sync_time,spatial_type)
SELECT id,'{2}',null,null,null,null,'{3}','{4}','{5}',{6},'{8}'
FROM {0}.monitor_slaveserver
WHERE name = '{1}';
END IF;
END$$;
"""
preview_file = task["status"].get_task_status("get_layer_preview").get_message("preview_file") or None
preview_file = "'{}'".format(preview_file) if preview_file else "null"
sql = sql_template.format(MASTER_PGSQL_SCHEMA, SLAVE_NAME,task['name'], task["job_id"], task["job_batch_id"], sync_message, "'{0}'".format(sync_time) if sync_time else 'null',preview_file,task.get('spatial_type',''))
#logger.info("Feature sync status notify: \r\n" + sql)
sql_cmd[len(sql_cmd) - 1] = sql
sql_process = subprocess.Popen(sql_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
sql_output = sql_process.communicate()
if sql_output[1] and sql_output[1].strip():
logger.info("stderr: {}".format(sql_output[1]))
if sql_process.returncode != 0 and sql_output[1].find("ERROR") >= 0:
cls._save_failed_sql(sql)
logger.error("Update sync status of task ({0}) in master db failed with return code ({1})".format(task['job_file'],sql_process.returncode))
except:
logger.error("Update sync status of task ({0}) in master db failed. {1}".format(task['job_file'],traceback.format_exc()))
def get_tasks(pull_status):
global ignore_files
changes = get_changeset()
next_job = False
for file_name, revision in changes.iteritems():
if DEBUG and INCLUDE and file_name not in INCLUDE:
#debug mode, file_name is not in INCLUDE
ignore_files += 1
continue
tasks = {}
logger.debug(
"Begin to check whether the file '{}' need synchronization or not."
.format(file_name))
action = ""
try:
segments = file_name.split('/', 2)
if revision in ['A', 'M']:
action = "update"
file_content = hg.cat([file_name], rev="tip")
elif revision == 'R':
action = "remove"
pre_rev = previous(HG_NODE)
try:
file_content = hg.cat([file_name], rev=pre_rev)
except:
#can't get the file content
logger.error("Can't get file '{}' content, ignore.".format(
file_name))
pull_status.get_task_status(file_name).set_message(
"message", "Failed to read file content, ignored.")
pull_status.get_task_status(file_name).set_message(
"action", action)
pull_status.get_task_status(file_name).succeed()
pull_status.get_task_status(
file_name).last_process_time = now()
continue
sync_job = parse_job(file_name, action, file_content)
action = sync_job["action"]
if action == 'none':
#no action is required
#pull_status.get_task_status(file_name).set_message("message","No action is required.")
#pull_status.get_task_status(file_name).set_message("action","None")
#pull_status.get_task_status(file_name).succeed()
#pull_status.get_task_status(file_name).last_process_time = now()
logger.debug(
"No action is required fot the file '{}', ignore. ".format(
file_name))
continue
logger.debug(
"The file '{}' is requested to perform '{}' action".format(
file_name, action))
sync_job["status"] = SlaveSyncStatus(file_name, action,
file_content)
#load meta data, if meta data is saved into a separated file
load_metafile(sync_job)
#convert bbox to array if bbox is a string
if "bbox" in sync_job and isinstance(sync_job["bbox"], basestring):
sync_job["bbox"] = json.loads(sync_job["bbox"])
#tasks will be added only after if a sync job has some unexecuted task or unsuccessful task.
job_failed = False
next_job = False
for task_type in ordered_sync_task_type:
if task_type not in sync_tasks_metadata: continue
if task_type not in sync_tasks: continue
for (task_metadata,
task_logger) in sync_tasks_metadata[task_type]:
try:
#if task_type == "update_access_rules":
# import ipdb;ipdb.set_trace()
if not is_sync_task(sync_job, segments, action,
task_metadata):
continue
if task_metadata[JOB_DEF_INDEX][CHANNEL_SUPPORT_INDEX]:
sync_job["channel"] = segments[0]
#check whether this task is already executed or not
if not job_failed and sync_job[
'status'].get_task_status(
task_type).is_succeed:
#this task is already succeed, continue
logger.debug(
"The task '{1}' is already done on the file '{0}',ignore"
.format(file_name, task_type))
break
#this task is not succeed or executed before, add this task to sync tasks
job_failed = True
task_name = taskname(sync_job, task_metadata)
if task_type not in tasks:
tasks[task_type] = {}
if task_name in sync_tasks[task_type]:
#task is already exist, this is a shared task
shared_task = sync_tasks[task_type][task_name]
if isinstance(shared_task, list):
task_status = shared_task[0][0][
'status'].get_task_status(task_type)
tasks[task_type][task_name] = shared_task + [
(sync_job, task_metadata, task_logger)
]
else:
task_status = shared_task[0][
'status'].get_task_status(task_type)
task_status.shared = True
tasks[task_type][task_name] = [
shared_task,
(sync_job, task_metadata, task_logger)
]
tasks[task_type][task_name] = sorted(
tasks[task_type][task_name],
key=lambda x: x[0]['job_file'],
reverse=True)
sync_job['status'].set_task_status(
task_type, task_status)
else:
#init a default status object for this task
sync_job['status'].get_task_status(task_type)
tasks[task_type][task_name] = (sync_job,
task_metadata,
task_logger)
#if task_type == "create_workspace": raise Exception("Failed for testing.")
break
except:
#preprocess the file failed, continue to the next file
message = traceback.format_exc()
logger.error(message)
tasks.clear()
sync_job['status'].get_task_status(
task_type).set_message(
"message",
"Preprocess the file failed. err = {0}".format(
message))
sync_job['status'].get_task_status(task_type).failed()
#this job is failed, try to add a notify task
for notify_metadata, notify_logger in notify_tasks_metadata:
if is_sync_task(sync_job, segments, action,
notify_metadata):
notify_tasks.append(
(sync_job, notify_metadata, notify_logger))
break
pull_status.get_task_status(file_name).set_message(
"action", action)
pull_status.get_task_status(file_name).set_message(
"message",
"Preprocess the file failed. err = {0}".format(
message))
pull_status.get_task_status(file_name).failed()
pull_status.get_task_status(
file_name).last_process_time = now()
next_job = True
break
if next_job:
break
if next_job:
continue
#add the sync job's tasks to the total sync tasks.
for key, val in tasks.iteritems():
sync_tasks[key].update(val)
if tasks:
#this job has some sync tasks to do,
#try to add a prepare task
for task_metadata, task_logger in prepare_tasks_metadata:
if is_sync_task(sync_job, segments, action, task_metadata):
prepare_tasks.append(
(sync_job, task_metadata, task_logger))
break
#try to add a notify task
for task_metadata, task_logger in notify_tasks_metadata:
if is_sync_task(sync_job, segments, action, task_metadata):
notify_tasks.append(
(sync_job, task_metadata, task_logger))
break
pull_status.get_task_status(file_name).set_message(
"message", "Ready to synchronize")
pull_status.get_task_status(file_name).set_message(
"action", action)
pull_status.get_task_status(file_name).succeed()
pull_status.get_task_status(
file_name).last_process_time = now()
else:
logger.debug(
"File({}) has been synchronized or no need to synchronize".
format(file_name))
except:
pull_status.get_task_status(file_name).failed()
message = traceback.format_exc()
pull_status.get_task_status(file_name).set_message(
"message", message)
pull_status.get_task_status(file_name).last_process_time = now()
logger.error("Add the '{1}' task for ({0}) failed.{2}".format(
file_name, action, traceback.format_exc()))
def get_tasks(pull_status):
global ignore_files
changes = get_changeset()
next_job = False
for file_name, revision in changes.iteritems():
if DEBUG and INCLUDE and file_name not in INCLUDE:
#debug mode, file_name is not in INCLUDE
ignore_files += 1
continue
tasks = {}
logger.debug("Begin to check whether the file '{}' need synchronization or not.".format(file_name))
action = ""
try:
segments = file_name.split('/',2)
if revision in ['A','M']:
action = "update"
file_content = hg.cat([file_name],rev="tip")
elif revision == 'R':
action = "remove"
pre_rev = previous(HG_NODE)
try:
file_content = hg.cat([file_name],rev=pre_rev)
except:
#can't get the file content
logger.error("Can't get file '{}' content, ignore.".format(file_name))
pull_status.get_task_status(file_name).set_message("message","Failed to read file content, ignored.")
pull_status.get_task_status(file_name).set_message("action",action)
pull_status.get_task_status(file_name).succeed()
pull_status.get_task_status(file_name).last_process_time = now()
continue
sync_job = parse_job(file_name,action,file_content)
action = sync_job["action"]
if action == 'none':
#no action is required
pull_status.get_task_status(file_name).set_message("message","No action is required.")
pull_status.get_task_status(file_name).set_message("action","None")
pull_status.get_task_status(file_name).succeed()
pull_status.get_task_status(file_name).last_process_time = now()
logger.debug("No action is required fot the file '{}', ignore. ".format(file_name))
continue
logger.debug("The file '{}' is requested to perform '{}' action".format(file_name,action))
sync_job["status"] = SlaveSyncStatus(file_name,action,file_content)
#load meta data, if meta data is saved into a separated file
load_metafile(sync_job)
#tasks will be added only after if a sync job has some unexecuted task or unsuccessful task.
job_failed = False
next_job = False
for task_type in ordered_sync_task_type:
if task_type not in sync_tasks_metadata: continue
if task_type not in sync_tasks:continue
for (task_metadata,task_logger) in sync_tasks_metadata[task_type]:
try:
#if task_type == "update_access_rules":
# import ipdb;ipdb.set_trace()
if not is_sync_task(sync_job,segments,action,task_metadata):
continue
if task_metadata[JOB_DEF_INDEX][CHANNEL_SUPPORT_INDEX]:
sync_job["channel"] = segments[0]
#check whether this task is already executed or not
if not job_failed and sync_job['status'].get_task_status(task_type).is_succeed:
#this task is already succeed, continue
logger.debug("The task '{1}' is already done on the file '{0}',ignore".format(file_name,task_type))
break
#this task is not succeed or executed before, add this task to sync tasks
job_failed = True
task_name = taskname(sync_job,task_metadata)
if task_type not in tasks:
tasks[task_type] = {}
if task_name in sync_tasks[task_type]:
#task is already exist, this is a shared task
shared_task = sync_tasks[task_type][task_name]
if isinstance(shared_task,list):
task_status = shared_task[0][0]['status'].get_task_status(task_type)
tasks[task_type][task_name] = shared_task + [(sync_job,task_metadata,task_logger)]
else:
task_status = shared_task[0]['status'].get_task_status(task_type)
task_status.shared = True
tasks[task_type][task_name] = [shared_task,(sync_job,task_metadata,task_logger)]
tasks[task_type][task_name] = sorted(tasks[task_type][task_name], key=lambda x: x[0]['job_file'], reverse=True)
sync_job['status'].set_task_status(task_type,task_status)
else:
#init a default status object for this task
sync_job['status'].get_task_status(task_type)
tasks[task_type][task_name] = (sync_job,task_metadata,task_logger)
#if task_type == "create_workspace": raise Exception("Failed for testing.")
break
except:
#preprocess the file failed, continue to the next file
message = traceback.format_exc()
logger.error(message)
tasks.clear()
sync_job['status'].get_task_status(task_type).set_message("message","Preprocess the file failed. err = {0}".format(message))
sync_job['status'].get_task_status(task_type).failed()
#this job is failed, try to add a notify task
for notify_metadata,notify_logger in notify_tasks_metadata:
if is_sync_task(sync_job,segments,action,notify_metadata):
notify_tasks.append((sync_job,notify_metadata,notify_logger))
break
pull_status.get_task_status(file_name).set_message("action",action)
pull_status.get_task_status(file_name).set_message("message","Preprocess the file failed. err = {0}".format(message))
pull_status.get_task_status(file_name).failed()
pull_status.get_task_status(file_name).last_process_time = now()
next_job = True
break
if next_job:
break
if next_job:
continue
#add the sync job's tasks to the total sync tasks.
for key,val in tasks.iteritems():
sync_tasks[key].update(val)
if tasks:
#this job has some sync tasks to do,
#try to add a prepare task
for task_metadata,task_logger in prepare_tasks_metadata:
if is_sync_task(sync_job,segments,action,task_metadata):
prepare_tasks.append((sync_job,task_metadata,task_logger))
break
#try to add a notify task
for task_metadata,task_logger in notify_tasks_metadata:
if is_sync_task(sync_job,segments,action,task_metadata):
notify_tasks.append((sync_job,task_metadata,task_logger))
break
pull_status.get_task_status(file_name).set_message("message","Ready to synchronize")
else :
pull_status.get_task_status(file_name).set_message("message","Already synchronized or no need to synchronize")
pull_status.get_task_status(file_name).set_message("action",action)
pull_status.get_task_status(file_name).succeed()
pull_status.get_task_status(file_name).last_process_time = now()
except:
pull_status.get_task_status(file_name).failed()
message = traceback.format_exc()
pull_status.get_task_status(file_name).set_message("message",message)
pull_status.get_task_status(file_name).last_process_time = now()
logger.error("Add the '{1}' task for ({0}) failed.{2}".format(file_name,action,traceback.format_exc()))
def get_bitbucket_status(cls):
if not hasattr(cls, "_bitbucket_status"):
cls._bitbucket_status = SlaveSyncStatus("bitbucket", "sync",
str(now()))
return cls._bitbucket_status
def get_bitbucket_status(cls):
if not hasattr(cls,"_bitbucket_status"):
cls._bitbucket_status = SlaveSyncStatus("bitbucket","sync",str(now()))
return cls._bitbucket_status
def load_metafile(sync_job):
meta_file = sync_job.get('meta', None)
if not meta_file:
#no meta file, all meta datas are embeded into the sync_job
return
task_status = sync_job['status'].get_task_status("load_metadata")
try:
if task_status.is_succeed:
#this task has been executed successfully,
#load the json file and add the meta data into sync_job
local_meta_file = task_status.get_message("meta_file")
with open(local_meta_file, "r") as f:
meta_data = json.loads(f.read())
sync_job.update(meta_data)
sync_job['meta']['local_file'] = local_meta_file
return
except:
pass
logger.info("Begin to load meta data for job({})".format(
sync_job['job_file']))
task_status.last_process_time = now()
if SHARE_LAYER_DATA:
sync_job['meta']['local_file'] = parse_remotefilepath(
sync_job["meta"]["file"])["file"]
meta_data = None
with open(sync_job['meta']['local_file'], "r") as f:
meta_data = json.loads(f.read())
sync_job.update(meta_data)
task_status.set_message("message",
"Find the meta file from shared layer data.")
task_status.set_message("meta_file", sync_job['meta']['local_file'])
task_status.succeed()
return
#download from borg master
temp_file = os.path.join(CACHE_PATH, "job.meta.json")
if sync_job['action'] == "remove":
#for remove action, doesn't check md5 because of the following case.
# 1. Publish an layer to repository
# 2. Slave pull from the repository and then publish the layer, at this time, the md5 of the layer's metadata file is 'A'
# 3. Publish the layer again, and now, the md5 of the layer's metadata file is 'B',
# 4. Remove the layer from repository.
# 5. Slave pull from the repository, only the last version will be fetched, and intermediate versions are ignored. so the publish action in step 3 is invisible for slave cient.
# 6. Slave client try to fetch the meta file from master and compare the md5 , and found: file's md5 is 'B', but md5 data in repository is 'A', doesn't match.
download_file(meta_file["file"], temp_file, task_status, None)
else:
download_file(meta_file["file"], temp_file, task_status,
meta_file.get('md5', None))
meta_data = None
with open(temp_file, "r") as f:
meta_data = json.loads(f.read())
sync_job.update(meta_data)
if "workspace" in sync_job:
local_meta_file = os.path.join(
CACHE_PATH, "{}.{}.meta.json".format(sync_job["workspace"],
sync_job["name"]))
else:
local_meta_file = os.path.join(CACHE_PATH,
"{}.meta.json".format(sync_job["name"]))
try:
os.remove(local_meta_file)
except:
#file not exist, ignore
pass
#rename to meta file
os.rename(temp_file, local_meta_file)
sync_job['meta']['local_file'] = local_meta_file
task_status.set_message("message",
"Succeed to download meta data from master.")
task_status.set_message("meta_file", local_meta_file)
task_status.succeed()
