logger.info("Executing {}...".format(repr(md5_cmd)))

get_md5 = subprocess.Popen(md5_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)

get_md5_output = get_md5.communicate()

if get_md5.returncode != 0:

raise Exception("{0}:{1}".format(get_md5.returncode,get_md5_output[1]))

elif get_md5_output[1] and get_md5_output[1].strip():

logger.info("stderr: {}".format(get_md5_output[1]))

if task_status:

task_status.set_message("message",get_md5_output[1])

file_md5 = get_md5_output[0].split()[0]

if file_md5 != md5:

if md5:

#check file md5 before downloading.

remote_file_path = remote_path

if remote_path.find("@") > 0:

#remote_path includes user@server prefix,remote that prefix

remote_file_path = remote_path.split(":",1)[1]

if SYNC_SERVER:

md5_cmd[len(md5_cmd) - 1] = remote_file_path

md5_cmd[len(md5_cmd) - 3] = SYNC_SERVER

check_file_md5(md5_cmd,md5,task_status)

elif remote_path.find("@") > 0:

md5_cmd[len(md5_cmd) - 1] = remote_file_path

md5_cmd[len(md5_cmd) - 3] = remote_path.split(":",1)[0]

check_file_md5(md5_cmd,md5,task_status)

else:

local_md5_cmd[len(local_md5_cmd) - 1] = remote_file_path

check_file_md5(local_md5_cmd,md5,task_status)

# sync over PostgreSQL dump with rsync

download_cmd[len(download_cmd) - 2] = remote_path

download_cmd[len(download_cmd) - 1] = local_path

logger.info("Executing {}...".format(repr(download_cmd)))

rsync = subprocess.Popen(download_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)

rsync_output = rsync.communicate()

if rsync.returncode != 0:

raise Exception("{0}:{1}".format(rsync.returncode,rsync_output[1]))

elif rsync_output[1] and rsync_output[1].strip():

logger.info("stderr: {}".format(rsync_output[1]))

if task_status:

task_status.set_message("message",rsync_output[1])

if md5:

#check file md5 after downloading

local_md5_cmd[len(local_md5_cmd) - 1] = local_path

meta_file = sync_job.get('meta',None)

if not meta_file:

#no meta file, all meta datas are embeded into the sync_job

return

task_status = sync_job['status'].get_task_status("load_metadata")

try:

if task_status.is_succeed:

#this task has been executed successfully,

#load the json file and add the meta data into sync_job

local_meta_file = task_status.get_message("meta_file")

with open(local_meta_file,"r") as f:

meta_data = json.loads(f.read())

sync_job.update(meta_data)

sync_job['meta']['local_file'] = local_meta_file

return

except:

pass

logger.info("Begin to load meta data for job({})".format(sync_job['job_file']))

task_status.last_process_time = now()

if SHARE_LAYER_DATA:

sync_job['meta']['local_file'] = parse_remotefilepath(sync_job["meta"]["file"])["file"]

meta_data = None

with open(sync_job['meta']['local_file'],"r") as f:

meta_data = json.loads(f.read())

sync_job.update(meta_data)

task_status.set_message("message","Find the meta file from shared layer data.")

task_status.set_message("meta_file",sync_job['meta']['local_file'])

task_status.succeed()

return

#download from borg master

temp_file = os.path.join(CACHE_PATH,"job.meta.json")

if sync_job['action'] == "remove":

#for remove action, doesn't check md5 because of the following case.

# 1. Publish an layer to repository

# 2. Slave pull from the repository and then publish the layer, at this time, the md5 of the layer's metadata file is 'A'

# 3. Publish the layer again, and now, the md5 of the layer's metadata file is 'B',

# 4. Remove the layer from repository.

# 5. Slave pull from the repository, only the last version will be fetched, and intermediate versions are ignored. so the publish action in step 3 is invisible for slave cient.

# 6. Slave client try to fetch the meta file from master and compare the md5 , and found: file's md5 is 'B', but md5 data in repository is 'A', doesn't match.

download_file(meta_file["file"],temp_file,task_status,None)

else:

download_file(meta_file["file"],temp_file,task_status,meta_file.get('md5',None))

meta_data = None

with open(temp_file,"r") as f:

meta_data = json.loads(f.read())

sync_job.update(meta_data)

if "workspace" in sync_job:

local_meta_file = os.path.join(CACHE_PATH,"{}.{}.meta.json".format(sync_job["workspace"],sync_job["name"]))

else:

local_meta_file = os.path.join(CACHE_PATH,"{}.meta.json".format(sync_job["name"]))

try:

os.remove(local_meta_file)

except:

#file not exist, ignore

pass

#rename to meta file

os.rename(temp_file,local_meta_file)

sync_job['meta']['local_file'] = local_meta_file

task_status.set_message("message","Succeed to download meta data from master.")

task_status.set_message("meta_file",local_meta_file)

if sync_job["action"] != "publish":

#not a publish job, no need to download table data

return

data_file = sync_job.get('data',None)

if not data_file:

raise Exception("Can't find data file in json file.")

if SYNC_SERVER:

#download from local slave

download_file("{0}:{1}/{2}.tar".format(SYNC_SERVER,SYNC_PATH,sync_job["name"]),data_file['local_file'],None,data_file.get("md5",None))

else:

#download from borg master

if sync_job["action"] == "remove":

#remove task, no need to download style file

return

style_files = sync_job.get('styles',None)

if not style_files:

return

for name,style_file in style_files.iteritems():

if SYNC_SERVER:

#download from local slave

if name == "builtin":

download_file("{}:{}/{}.sld".format(SYNC_SERVER,SYNC_PATH,sync_job["name"]),style_file['local_file'],task_status,style_file.get("md5",None))

else:

download_file("{}:{}/{}.{}.sld".format(SYNC_SERVER,SYNC_PATH,sync_job["name"],name),style_file['local_file'],task_status,style_file.get("md5",None))

task_status.set_message("message","Succeed to download style file from slave server {0}".format(SYNC_SERVER))

else:

#download from borg master

download_file(style_file["file"],style_file['local_file'],task_status,style_file.get("md5",None))

# sync over PostgreSQL dump with rsync

upload_cmd[len(upload_cmd) - 2] = local_file

upload_cmd[len(upload_cmd) - 1] = remote_path

logger.info("Executing {}...".format(repr(upload_cmd)))

rsync = subprocess.Popen(upload_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)

rsync_output = rsync.communicate()

if rsync_output[1] and rsync_output[1].strip():

logger.info("stderr: {}".format(rsync_output[1]))

task_status.set_message("message",rsync_output[1])

if rsync.returncode != 0:

if sync_job["status"].get_task_status("get_layer_preview").get_message("preview_file"):

local_file = os.path.join(PREVIEW_ROOT_PATH,".",sync_job["status"].get_task_status("get_layer_preview").get_message("preview_file"))

upload_file(local_file,sync_job["preview_path"],task_status)

task_status.set_message("message","Upload file {} to {}".format(local_file,sync_job["preview_path"]))

else:

task_status.task_failed()

"""

Delete the dump files

1. table data file

2. meta file

3. style files

"""

messages = []

for f in [local_file for local_file in (

[sync_job.get('data',{}).get('local_file')] +

[sync_job.get('meta',{}).get('local_file')] +

[style_file.get('local_file') for style_file in (sync_job.get('styles') or {}).itervalues() ]

) if local_file ]:

try:

if os.path.exists(f):

os.remove(f)

messages.append("Succeed to remove file({}).".format(f))

except:

message = traceback.format_exc()

logger.error("Remove file ({}) failed. {}".format(f,message))

messages.append("Failed to remove file({}). {}".format(f,message))

task_status.task_failed()