def delete(self, job_id):
JobsLoader.Instance().delete_job(job_id)
scheduler = PydioScheduler.Instance()
scheduler.reload_configs()
scheduler.disable_job(job_id)
JobsLoader.Instance().clear_job_data(job_id, parent=True)
return job_id + "deleted", 204
def get(self, job_id='', relative_path=''):
"""
retrieves the stat info for a given file / list the active job details
:param job_id: (optional) Job Id of the file/ folder
:param relative_path: (optional) relative path of the file/folder with respect
to the corresponding repository(job_id)
:returns a json response
"""
if request.path == '/stat':
jobs = JobsLoader.Instance().get_jobs()
json_jobs = {}
for job in jobs:
json_jobs.update({jobs[job].id: [jobs[job].directory, jobs[job].server, jobs[job].label, jobs[job].workspace]})
return json_jobs
else:
directory_path = JobsLoader.Instance().get_job(job_id).directory
base_path = JobsLoader.Instance().build_job_data_path(job_id)
path = os.path.join(directory_path, relative_path)
#r = os.stat(path)
# Get the status of the file idle/busy... by join of ajxp_index and ajxp_node_status tables
db_handler = LocalDbHandler(base_path, directory_path)
if Path(str(path.encode("utf-8"))).is_dir():
node_status = db_handler.get_directory_node_status("/" + relative_path)
else:
node_status = db_handler.get_node_status("/" + relative_path)
return {"node_status": node_status}
def get(self, job_id):
if not job_id in JobsLoader.Instance().get_jobs():
return "Can't find any job config with this ID.", 404
dbHandler = LocalDbHandler(
JobsLoader.Instance().build_job_data_path(job_id))
return dbHandler.list_conflict_nodes()
def post(self):
JobsLoader.Instance().get_jobs()
json_req = request.get_json()
new_job = JobConfig.object_decoder(json_req)
if 'test_path' in json_req:
json_req['directory'] = os.path.join(
ConfigManager.Instance().get_data_path(),
json_req['repoObject']['label'])
return json_req
elif 'compute_sizes' in json_req:
dl_rate = 2 * 1024 * 1024
up_rate = 0.1 * 1024 * 1024
# COMPUTE REMOTE SIZE
from pydio.sdk.remote import PydioSdk
trust_ssl = False
if 'trust_ssl' in json_req:
trust_ssl = json_req['trust_ssl']
sdk = PydioSdk(
json_req['server'],
json_req['workspace'],
json_req['remote_folder'],
'',
auth=(json_req['user'], json_req['password']),
device_id=ConfigManager.Instance().get_device_id(),
skip_ssl_verify=trust_ssl,
proxies=ConfigManager.Instance().get_defined_proxies())
up = [0.0]
def callback(location, change, info):
if change and "bytesize" in change and change[
"md5"] != "directory":
up[0] += float(change["bytesize"])
sdk.changes_stream(0, callback)
# COMPUTE LOCAL SIZE
down = 0.0
if os.path.exists(json_req['directory']):
for dirpath, dirnames, filenames in os.walk(
json_req['directory']):
for f in filenames:
fp = os.path.join(dirpath, f)
try:
down += os.path.getsize(fp)
except OSError:
pass
json_req['byte_size'] = up[0] + down
json_req['eta'] = up[0] * 8 / dl_rate + down * 8 / up_rate
return json_req
JobsLoader.Instance().update_job(new_job)
scheduler = PydioScheduler.Instance()
scheduler.reload_configs()
scheduler.disable_job(new_job.id)
if not 'toggle_status' in json_req:
JobsLoader.Instance().clear_job_data(new_job.id)
scheduler.enable_job(new_job.id)
return JobConfig.encoder(new_job)
def get(self, cmd, job_id=None):
if job_id:
if cmd == 'enable' or cmd == 'disable':
job_config = JobsLoader.Instance().get_job(job_id)
job_config.active = True if cmd == 'enable' else False
JobsLoader.Instance().update_job(job_config)
PydioScheduler.Instance().reload_configs()
PydioScheduler.Instance().handle_job_signal(self, cmd, job_id)
else:
return PydioScheduler.Instance().handle_generic_signal(self, cmd)
return ('success', )
def get(self, job_id):
if not job_id in JobsLoader.Instance().get_jobs():
return "Can't find any job config with this ID.", 404
logger = EventLogger(JobsLoader.Instance().build_job_data_path(job_id))
if not request.args:
logs = logger.get_all(20, 0)
else:
filter = request.args.keys()[0]
filter_parameter = request.args.get(filter)
logs = logger.filter(filter, filter_parameter)
tasks = PydioScheduler.Instance().get_job_progress(job_id)
return {"logs": logs, "running": tasks}
def get(self, job_id):
if job_id != 'request':
jobs = JobsLoader.Instance().get_jobs()
if not job_id in jobs:
return {"error": "Cannot find job"}
job = jobs[job_id]
url = job.server + '/api/' + job.workspace + '/ls/?options=d&recursive=true'
auth = (job.user_id, keyring.get_password(job.server, job.user_id))
verify = not job.trust_ssl
else:
args = request.args
base = args['url'].rstrip('/')
verify = False if args['trust_ssl'] == 'true' else True
url = base + '/api/' + args[
'ws'] + '/ls/?options=d&recursive=true&max_depth=2'
if 'password' in args:
auth = (args['user'], args['password'])
else:
auth = (args['user'], keyring.get_password(base, args['user']))
if verify and "REQUESTS_CA_BUNDLE" in os.environ:
verify = os.environ["REQUESTS_CA_BUNDLE"]
resp = requests.get(url, stream=True, auth=auth, verify=verify)
o = xmltodict.parse(resp.content)
if not 'tree' in o or 'message' in o['tree']:
return [{'error': 'Cannot load workspace'}]
if not 'tree' in o['tree']:
return []
if isinstance(o['tree']['tree'], types.DictType):
return [o['tree']['tree']]
return o['tree']['tree']
def get(self, job_id):
jobs = JobsLoader.Instance().get_jobs()
if not job_id in jobs:
return {"error": "Cannot find job"}
try:
from pydio.job.change_history import ChangeHistory
except ImportError:
from job.change_history import ChangeHistory
scheduler = PydioScheduler.Instance()
job = scheduler.control_threads[job_id]
args = request.args
res = ""
if 'status' in args:
if args['status'].upper() == 'SUCCESS':
for failed in job.current_store.change_history.get_all_success(
):
res += failed
elif args['status'].upper() == 'FAILED':
for failed in job.current_store.change_history.get_all_failed(
):
res += failed
else:
return {
'error': "Unknown status: " + urllib2.quote(args['status'])
}
else:
for failed in job.current_store.change_history.get_all():
res += failed
return res
def start_watcher(self):
if self.watcher:
if self.watcher_first_run:
def status_callback(status):
self.logger.log_state(status, 'sync')
self.init_global_progress()
try:
self.global_progress['status_indexing'] = 1
self.logger.log_state(
_('Checking changes since last launch...'), "sync")
very_first = True
self.db_handler.update_bulk_node_status_as_idle()
self.watcher.check_from_snapshot(
state_callback=status_callback)
except DBCorruptedException as e:
self.stop()
JobsLoader.Instance().clear_job_data(self.job_config.id)
logging.error(e)
return
except Exception as e:
logging.exception(e)
self.interrupt = True
self.logger.log_state(
_('Oops, error while indexing the local folder. Pausing the task.'
), 'error')
logging.error(e)
self.watcher_first_run = False
self.watcher.start()
def get(self, job_id):
if job_id != 'request':
jobs = JobsLoader.Instance().get_jobs()
if not job_id in jobs:
return {"error": "Cannot find job"}
job = jobs[job_id]
url = job.server + '/api/pydio/state/user/repositories?format=json'
auth = (job.user_id, keyring.get_password(job.server, job.user_id))
verify = not job.trust_ssl
else:
args = request.args
base = args['url'].rstrip('/')
verify = False if args['trust_ssl'] == 'true' else True
url = base + '/api/pydio/state/user/repositories?format=json'
if 'password' in args:
auth = (args['user'], args['password'])
else:
auth = (args['user'], keyring.get_password(base, args['user']))
if verify and "REQUESTS_CA_BUNDLE" in os.environ:
verify = os.environ["REQUESTS_CA_BUNDLE"]
resp = requests.get(url, stream=True, auth=auth, verify=verify)
data = json.loads(resp.content)
if 'repositories' in data and 'repo' in data['repositories']:
if isinstance(data['repositories']['repo'], types.DictType):
data['repositories']['repo'] = [data['repositories']['repo']]
data['repositories']['repo'] = filter(
lambda x: not x['@access_type'].startswith('ajxp_'),
data['repositories']['repo'])
return data
def post(self):
json_conflict = request.get_json()
job_id = json_conflict['job_id']
try:
job_config = JobsLoader.Instance().get_job(job_id)
except Exception:
return "Can't find any job config with this ID.", 404
dbHandler = LocalDbHandler(
JobsLoader.Instance().build_job_data_path(job_id))
dbHandler.update_node_status(json_conflict['node_path'],
json_conflict['status'])
if not dbHandler.count_conflicts() and job_config.active:
t = PydioScheduler.Instance().get_thread(job_id)
if t:
t.start_now()
return json_conflict
def enrich_job(self, job_data, job_id, get_notification=False):
running = PydioScheduler.Instance().is_job_running(job_id)
job_data['running'] = running
logger = EventLogger(JobsLoader.Instance().build_job_data_path(job_id))
if get_notification:
notification = logger.consume_notification()
if notification:
job_data['notification'] = notification
last_events = logger.get_all(1, 0)
if len(last_events):
job_data['last_event'] = last_events.pop()
if running:
job_data['state'] = PydioScheduler.Instance().get_job_progress(job_id)
try:
job_data['last_action'] = EventLogger(JobsLoader.Instance().build_job_data_path(job_id)).get_last_action()[-1][-1]
except IndexError:
pass
def enrich_job(self, job_data, job_id):
running = PydioScheduler.Instance().is_job_running(job_id)
job_data['running'] = running
logger = EventLogger(JobsLoader.Instance().build_job_data_path(job_id))
last_events = logger.get_all(1, 0)
if len(last_events):
job_data['last_event'] = last_events.pop()
if running:
job_data['state'] = PydioScheduler.Instance().get_job_progress(
job_id)
def get(self, job_id):
"""
retrieves the stat info for a given file / list the active job details
:param job_id: (optional) Job Id of the file/ folder that needs to be shared
:returns a json response
on success: returns a shared link
"""
args = request.args
jobs = JobsLoader.Instance().get_jobs()
if not job_id in jobs:
return {"error": "Cannot find job"}
job = jobs[job_id]
from pydio.sdk.remote import PydioSdk
remote_instance = PydioSdk(job.server, job.workspace, job.remote_folder, job.user_id,
auth="",
device_id=ConfigManager.Instance().get_device_id(),
skip_ssl_verify=job.trust_ssl,
proxies=ConfigManager.Instance().get_defined_proxies(),
timeout=job.timeout)
if args['action'] == 'share':
relative_path = os.path.normpath(job.remote_folder + "/" + args["relative_path"]).replace('\\', '/')
# Check if the shared link is already present
check_res = remote_instance.check_share_link(
relative_path
)
if len(check_res) > 2: # when share link doesn't exists content length will be zero for file and 2 for folder
res = json.loads(check_res)
if "minisite" in res and res["minisite"]["public"]:
return {"link": res["minisite"]["public_link"], "existingLinkFlag": "true"}
elif "repositoryId" in res:
return {"link": _("The folder is already shared as a workspace!"), "existingLinkFlag": "true"}
elif args["checkExistingLinkFlag"]:
return {"existingLinkFlag": "false"}
res = remote_instance.share(
args["ws_label"],
args["ws_description"] if "ws_description" in args else "",
args["password"] if "password" in args else "",
args["expiration"] if "expiration" in args else 0,
args["downloads"] if "downloads" in args else 0,
args["can_read"] if "can_read" in args else "true",
args["can_download"] if "can_download" in args else "true",
relative_path,
args["link_handler"] if "link_handler" in args else "",
args["can_write"] if "can_write" in args else "false"
)
return {"link": res}
else:
res = remote_instance.unshare(job.remote_folder + "/" + args["path"])
return {"response": res, "existingLinkFlag": "false"}
def __init__(self, server_port, user, password, external_ip=None):
logging.info('-----------------------------------------------')
if external_ip:
logging.info('Starting agent on http://' + external_ip + ':' +
str(server_port) + '/')
logging.info('Warning, this agent UI is world accessible!')
else:
logging.info('Starting agent locally on http://localhost:' +
str(server_port) + '/')
logging.info('------------------------------------------------')
self.user_data_path = JobsLoader.Instance().data_path
self.port = server_port
self.external_ip = external_ip
authDB.add_user(user, password)
self.running = False
if getattr(sys, 'frozen', False):
self.real_static_folder = Path(sys._MEIPASS) / 'ui' / 'res'
static_folder = str(self.real_static_folder)
else:
self.real_static_folder = Path(__file__).parent / 'res'
static_folder = 'res'
logging.debug('Starting Flask server with following static folder : ' +
static_folder)
self.app = Flask(__name__,
static_folder=static_folder,
static_url_path='/res')
self.app.logger.setLevel(logging.ERROR)
l = logging.getLogger("werkzeug")
if l:
l.setLevel(logging.ERROR)
super(PydioApi, self).__init__(self.app)
self.add_resource(JobManager, '/', '/jobs', '/jobs/<string:job_id>',
'/jobs-status')
self.add_resource(WorkspacesManager, '/ws/<string:job_id>')
self.add_resource(FoldersManager, '/folders/<string:job_id>')
self.add_resource(LogManager, '/jobs/<string:job_id>/logs')
self.add_resource(ConflictsManager, '/jobs/<string:job_id>/conflicts',
'/jobs/conflicts')
self.add_resource(CmdManager, '/cmd/<string:cmd>/<string:job_id>',
'/cmd/<string:cmd>')
self.add_resource(ProxyManager, '/proxy')
self.app.add_url_rule('/res/i18n.js', 'i18n', self.serve_i18n_file)
self.app.add_url_rule('/res/config.js', 'config',
self.server_js_config)
self.app.add_url_rule('/res/dynamic.css', 'dynamic_css',
self.serve_dynamic_css)
self.app.add_url_rule('/res/about.html', 'dynamic_about',
self.serve_about_content)
if EndpointResolver:
self.add_resource(ResolverManager, '/resolve/<string:client_id>')
self.app.add_url_rule('/res/dynamic.png', 'dynamic_png',
self.serve_dynamic_image)
def get(self, job_id=None):
if request.path == '/':
return redirect("/res/index.html", code=302)
jobs = JobsLoader.Instance().get_jobs()
if not job_id:
std_obj = []
for k in jobs:
data = JobConfig.encoder(jobs[k])
self.enrich_job(data, k)
std_obj.append(data)
return std_obj
data = JobConfig.encoder(jobs[job_id])
self.enrich_job(data, job_id)
return data
def get(self, job_id = None):
if request.path == '/':
return redirect("/res/index.html", code=302)
jobs = JobsLoader.Instance().get_jobs()
if not job_id:
json_jobs = []
for k in jobs:
data = JobConfig.encoder(jobs[k])
self.enrich_job(data, k, (request.path == '/jobs-status'))
json_jobs.append(data)
if request.path == '/jobs-status':
response = {'is_connected_to_internet': connection_helper.internet_ok, 'jobs': json_jobs}
return response
return json_jobs
logging.info("Requiring job %s" % job_id)
data = JobConfig.encoder(jobs[job_id])
self.enrich_job(data, job_id)
return data
def check_sync(self, job_id=None):
# load conf
conf = JobsLoader.Instance()
jobs = conf.jobs
if job_id is None:
return Response(str(jobs.keys()), status=200, mimetype="text")
if job_id not in jobs:
return Response("Unknown job", status=400, mimetype="text")
# check job exists
job = jobs[job_id]
sdk = PydioSdk(job.server,
ws_id=job.workspace,
remote_folder=job.remote_folder,
user_id=job.user_id,
device_id=ConfigManager.Instance().get_device_id(),
skip_ssl_verify=job.trust_ssl,
proxies=ConfigManager.Instance().get_defined_proxies(),
timeout=380)
checker = SyncChecker(job_id, jobs, sdk)
resp = checker.dofullcheck()
return Response(json.dumps(resp), status=200, mimetype="text/json")
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser('Pydio Synchronization Tool')
parser.add_argument('-s',
'--server',
help='Server URL, with http(s) and path to pydio',
type=unicode,
default='http://localhost')
parser.add_argument('-d',
'--directory',
help='Local directory',
type=unicode,
default=None)
parser.add_argument('-w',
'--workspace',
help='Id or Alias of workspace to synchronize',
type=unicode,
default=None)
parser.add_argument(
'-r',
'--remote_folder',
help='Path to an existing folder of the workspace to synchronize',
type=unicode,
default=None)
parser.add_argument('-u',
'--user',
help='User name',
type=unicode,
default=None)
parser.add_argument('-p',
'--password',
help='Password',
type=unicode,
default=None)
parser.add_argument('-dir',
'--direction',
help='Synchro Direction',
type=str,
default='bi')
parser.add_argument('-f',
'--file',
type=unicode,
help='Json file containing jobs configurations')
parser.add_argument(
'-z',
'--zmq_port',
type=int,
help=
'Available port for zmq, both this port and this port +1 will be used',
default=5556)
parser.add_argument('-i',
'--rdiff',
type=unicode,
help='Path to rdiff executable',
default=None)
parser.add_argument('--diag',
help='Run self diagnostic',
action='store_true',
default=False)
parser.add_argument('--diag-http',
help='Check server connection',
action='store_true',
default=False)
parser.add_argument('--diag-imports',
help='Check imports and exit',
action='store_true',
default=False)
parser.add_argument('--save-cfg', action='store_true', default=True)
parser.add_argument(
'--extract_html',
help='Utils for extracting HTML strings and compiling po files to json',
type=unicode,
default=False)
parser.add_argument('--auto-start', action='store_true')
parser.add_argument('--auto_detect_port',
type=bool,
help='Auto detect available ports',
default=False)
parser.add_argument('-v', '--verbose', action='count', default=1)
args, _ = parser.parse_known_args(argv)
jobs_root_path = Path(__file__).parent / 'data'
if not jobs_root_path.exists():
jobs_root_path = Path(DEFAULT_DATA_PATH)
if not jobs_root_path.exists():
jobs_root_path.mkdir()
setup_logging(args.verbose, jobs_root_path)
if args.auto_start:
import pydio.autostart
pydio.autostart.setup(argv)
return 0
jobs_loader = JobsLoader.Instance(data_path=str(jobs_root_path))
config_manager = ConfigManager.Instance(data_path=str(jobs_root_path))
config_manager.set_rdiff_path(args.rdiff)
if args.server and args.directory and args.workspace:
job_config = JobConfig()
job_config.load_from_cliargs(args)
data = {job_config.id: job_config}
if args.save_cfg:
logging.info("Storing config in %s",
str(jobs_root_path / 'configs.json'))
jobs_loader.save_jobs(data)
else:
fp = args.file
if fp and fp != '.':
logging.info("Loading config from %s", fp)
jobs_loader.config_file = fp
jobs_loader.load_config()
data = jobs_loader.get_jobs()
logging.debug("data: %s" %
json.dumps(data, default=JobConfig.encoder, indent=2))
if args.diag_imports:
# nothing more to do
return sys.exit(0)
if args.extract_html:
from pydio.utils.i18n import PoProcessor
proc = PoProcessor()
if args.extract_html == 'extract':
root = Path(__file__).parent
count = proc.extract_all_html_strings(
str(root / 'ui' / 'res'),
str(root / 'res' / 'i18n' / 'html_strings.py'))
logging.info(
'Wrote %i strings to html_strings.py - Now update PO files using standard tools'
% count)
# nothing more to do
elif args.extract_html == 'compile':
root = Path(__file__).parent
proc.po_to_json(str(root / 'res' / 'i18n' / '*.po'),
str(root / 'ui' / 'res' / 'i18n.js'))
return sys.exit(0)
if args.diag_http:
keys = data.keys()
if args.password:
smoke_tests = PydioDiagnostics(data[keys[0]].server,
data[keys[0]].workspace,
data[keys[0]].remote_folder,
data[keys[0]].user_id,
args.password)
else:
smoke_tests = PydioDiagnostics(data[keys[0]].server,
data[keys[0]].workspace,
data[keys[0]].remote_folder,
data[keys[0]].user_id)
rc = smoke_tests.run()
if rc != 0:
logging.error("Diagnostics failed: %s %s" %
(str(rc), smoke_tests.status_message))
return sys.exit(rc)
ports_detector = PortsDetector(args.zmq_port,
args.auto_detect_port,
store_file=str(jobs_root_path /
'ports_config'))
ports_detector.create_config_file()
scheduler = PydioScheduler.Instance(jobs_root_path=jobs_root_path,
jobs_loader=jobs_loader)
server = PydioApi(ports_detector.get_open_port('flask_api'))
from pydio.job import manager
manager.api_server = server
try:
thread.start_new_thread(server.start_server, ())
time.sleep(0.3)
if not server.running:
logging.error('Cannot start web server, exiting application')
sys.exit(1)
scheduler.start_all()
except (KeyboardInterrupt, SystemExit):
server.shutdown_server()
sys.exit()
def get(self, job_id):
if job_id != 'request':
jobs = JobsLoader.Instance().get_jobs()
if not job_id in jobs:
return {"error": "Cannot find job"}
job = jobs[job_id]
url = job.server + '/api/pydio/state/user/repositories?format=json'
auth = (job.user_id, keyring.get_password(job.server, job.user_id))
verify = not job.trust_ssl
else:
args = request.args
base = args['url'].rstrip('/')
verify = False if args['trust_ssl'] == 'true' else True
url = base + '/api/pydio/state/user/repositories?format=json'
if 'password' in args:
auth = (args['user'], args['password'])
else:
auth = (args['user'], keyring.get_password(base, args['user']))
app_name_url = base + '/api/pydio/state/plugins?format=json'
display_name_url = base + '/api/pydio/state/user/preferences?format=json'
if verify and "REQUESTS_CA_BUNDLE" in os.environ:
verify = os.environ["REQUESTS_CA_BUNDLE"]
try:
# TRY TO GET APPLICATION TITLE
if app_name_url:
resp = requests.get(
app_name_url,
stream=False,
auth=auth,
verify=verify,
proxies=ConfigManager.Instance().get_defined_proxies())
resp.raise_for_status()
try:
app_data = json.loads(resp.content)
app_name = ''
ajxpcores = app_data['plugins']['ajxpcore']
for core in ajxpcores:
if core['@id'] == 'core.ajaxplorer':
for prop in core['plugin_configs']['property']:
if prop['@name'] == 'APPLICATION_TITLE':
app_name = json.loads(prop['$'])
break
break
except KeyError as k:
pass
except ValueError:
pass
# TRY TO GET USER DISPLAY NAME
if display_name_url:
resp = requests.get(
display_name_url,
stream=False,
auth=auth,
verify=verify,
proxies=ConfigManager.Instance().get_defined_proxies())
resp.raise_for_status()
try:
user_data = json.loads(resp.content)
user_display_name = ''
prefs = user_data['preferences']['pref']
for pref in prefs:
if pref['@name'] == 'USER_DISPLAY_NAME':
if pref['@value']:
user_display_name = pref['@value']
break
except KeyError as k:
pass
except ValueError:
pass
resp = requests.get(
url,
stream=True,
auth=auth,
verify=verify,
proxies=ConfigManager.Instance().get_defined_proxies())
resp.raise_for_status()
data = json.loads(resp.content)
if 'repositories' in data and 'repo' in data['repositories']:
if isinstance(data['repositories']['repo'], types.DictType):
data['repositories']['repo'] = [
data['repositories']['repo']
]
data['repositories']['repo'] = filter(
lambda x: not x['@access_type'].startswith('ajxp_'),
data['repositories']['repo'])
if app_name:
data['application_title'] = app_name
if user_display_name:
data['user_display_name'] = user_display_name
return data
except requests.HTTPError:
r = resp.status_code
message = _("Couldn't load your workspaces, check your server !")
if r == 404:
message = _(
"Server not found (404), is it up and has it Pydio installed ?"
)
elif r == 401:
message = _(
"Authentication failed: please verify your login and password"
)
elif r == 403:
message = _("Access to the server is forbidden")
elif r == 500 or r == 408:
message = _("Server seems to be encountering problems (500)")
logging.debug("Error while loading workspaces : " + message)
return {'error': message}, resp.status_code
except SSLError as rt:
logging.error(rt.message)
return {
'error':
_("An SSL error happened! Is your server using a self-signed certificate? In that case please check 'Trust SSL certificate'"
)
}, 400
except ProxyError as rt:
logging.error(rt.message)
return {
'error': _('A proxy error happened, please check the logs')
}, 400
except TooManyRedirects as rt:
logging.error(rt.message)
return {'error': _('Connection error: too many redirects')}, 400
except ChunkedEncodingError as rt:
logging.error(rt.message)
return {
'error': _('Chunked encoding error, please check the logs')
}, 400
except ContentDecodingError as rt:
logging.error(rt.message)
return {
'error': _('Content Decoding error, please check the logs')
}, 400
except InvalidSchema as rt:
logging.error(rt.message)
return {'error': _('Http connection error: invalid schema.')}, 400
except InvalidURL as rt:
logging.error(rt.message)
return {'error': _('Http connection error: invalid URL.')}, 400
except ValueError:
message = "Error while parsing request result:" + resp.content
logging.debug(message)
return {'error': message}, 400
except Timeout as to:
logging.error(to)
return {'error': _('Connection timeout!')}, 400
except RequestException as ree:
logging.error(ree.message)
return {'error': _('Cannot resolve domain!')}, 400
def run(self):
"""
Start the thread
"""
logger = EventLogger(self.configs_path)
very_first = False
if self.watcher:
if self.watcher_first_run:
def status_callback(status):
logger.log_state(status, 'sync')
self.init_global_progress()
try:
self.global_progress['status_indexing'] = 1
logger.log_state(
_('Checking changes since last launch...'), "sync")
very_first = True
self.watcher.check_from_snapshot(
state_callback=status_callback)
except DBCorruptedException as e:
self.stop()
JobsLoader.Instance().clear_job_data(self.job_config.id)
logging.error(e)
return
except Exception as e:
self.interrupt = True
logger.log_state(
_('Oops, error while indexing the local folder. Pausing the task.'
), 'error')
logging.error(e)
self.watcher_first_run = False
self.watcher.start()
while not self.interrupt:
try:
# logging.info('Starting cycle with cycles local %i and remote %is' % (self.local_seq, self.remote_seq))
self.processing_signals = {}
self.init_global_progress()
if very_first:
self.global_progress['status_indexing'] = 1
interval = int(time.time() - self.last_run)
if (self.online_status and interval < self.online_timer) or (
not self.online_status
and interval < self.offline_timer):
time.sleep(self.event_timer)
continue
if not self.job_status_running:
logging.debug("self.online_timer: %s" % self.online_timer)
logger.log_state(_('Status: Paused'), "sync")
self.sleep_offline()
continue
if self.job_config.frequency == 'time':
start_time = datetime.time(
int(self.job_config.start_time['h']),
int(self.job_config.start_time['m']))
end_time = datetime.time(
int(self.job_config.start_time['h']),
int(self.job_config.start_time['m']), 59)
now = datetime.datetime.now().time()
if not start_time < now < end_time:
logger.log_state(
_('Status: scheduled for %s') % str(start_time),
"sync")
self.sleep_offline()
continue
else:
logging.info(
"Now triggering synchro as expected at time " +
str(start_time))
if not self.system.check_basepath():
log = _(
'Cannot find local folder! Did you disconnect a volume? Waiting %s seconds before retry'
) % self.offline_timer
logging.error(log)
logger.log_state(
_('Cannot find local folder, did you disconnect a volume?'
), "error")
self.sleep_offline()
continue
if self.watcher:
for snap_path in self.marked_for_snapshot_pathes:
logging.info(
'LOCAL SNAPSHOT : loading snapshot for directory %s'
% snap_path)
if self.interrupt or not self.job_status_running:
raise InterruptException()
self.watcher.check_from_snapshot(snap_path)
# Load local and/or remote changes, depending on the direction
from pydio.job.change_stores import SqliteChangeStore
self.current_store = SqliteChangeStore(
self.configs_path + '/changes.sqlite',
self.job_config.filters['includes'],
self.job_config.filters['excludes'])
self.current_store.open()
try:
if self.job_config.direction != 'up':
logging.info(
'Loading remote changes with sequence %s' %
str(self.remote_seq))
if self.remote_seq == 0:
logger.log_state(
_('Gathering data from remote workspace, this can take a while...'
), 'sync')
very_first = True
self.remote_target_seq = self.load_remote_changes_in_store(
self.remote_seq, self.current_store)
self.current_store.sync()
else:
self.remote_target_seq = 1
self.ping_remote()
except RequestException as ce:
if not connection_helper.is_connected_to_internet():
error = _(
'No Internet connection detected! Waiting for %s seconds to retry'
) % self.offline_timer
else:
error = _(
'Connection to server failed, server is probably down. Waiting %s seconds to retry'
) % self.offline_timer
self.marked_for_snapshot_pathes = []
logging.error(error)
logger.log_state(error, "wait")
self.sleep_offline()
continue
except Exception as e:
error = 'Error while connecting to remote server (%s), waiting for %i seconds before retempting ' % (
e.message, self.offline_timer)
logging.error(error)
logger.log_state(
_('Error while connecting to remote server (%s)') %
e.message, "error")
self.marked_for_snapshot_pathes = []
self.sleep_offline()
continue
self.online_status = True
if not self.job_config.server_configs:
self.job_config.server_configs = self.sdk.load_server_configs(
)
self.sdk.set_server_configs(self.job_config.server_configs)
if self.job_config.direction != 'down':
logging.info('Loading local changes with sequence ' +
str(self.local_seq))
self.local_target_seq = self.db_handler.get_local_changes_as_stream(
self.local_seq, self.current_store.flatten_and_store)
self.current_store.sync()
else:
self.local_target_seq = 1
if not connection_helper.internet_ok:
connection_helper.is_connected_to_internet()
changes_length = len(self.current_store)
if not changes_length:
logging.info('No changes detected')
self.update_min_seqs_from_store()
self.exit_loop_clean(logger)
very_first = False
continue
self.global_progress['status_indexing'] = 1
logging.info('Reducing changes')
logger.log_state(
_('Merging changes between remote and local, please wait...'
), 'sync')
logging.debug('Delete Copies')
self.current_store.delete_copies()
self.update_min_seqs_from_store()
logging.debug('Dedup changes')
self.current_store.dedup_changes()
self.update_min_seqs_from_store()
if not self.storage_watcher or very_first:
logging.debug('Detect unnecessary changes')
self.current_store.detect_unnecessary_changes(
local_sdk=self.system, remote_sdk=self.sdk)
self.update_min_seqs_from_store()
logging.debug('Clearing op and pruning folders moves')
self.current_store.clear_operations_buffer()
self.current_store.prune_folders_moves()
self.update_min_seqs_from_store()
logging.debug('Store conflicts')
store_conflicts = self.current_store.clean_and_detect_conflicts(
self.db_handler)
if store_conflicts:
logging.info('Conflicts detected, cannot continue!')
logger.log_state(_('Conflicts detected, cannot continue!'),
'error')
self.current_store.close()
self.sleep_offline()
continue
changes_length = len(self.current_store)
if not changes_length:
logging.info('No changes detected')
self.exit_loop_clean(logger)
very_first = False
continue
self.global_progress['status_indexing'] = 0
import change_processor
self.global_progress['queue_length'] = changes_length
logging.info('Processing %i changes' % changes_length)
logger.log_state(
_('Processing %i changes') % changes_length, "start")
counter = [1]
def processor_callback(change):
try:
if self.interrupt or not self.job_status_running:
raise InterruptException()
self.update_current_tasks()
self.update_global_progress()
Processor = StorageChangeProcessor if self.storage_watcher else ChangeProcessor
proc = Processor(change, self.current_store,
self.job_config, self.system,
self.sdk, self.db_handler,
self.event_logger)
proc.process_change()
self.update_min_seqs_from_store(success=True)
self.global_progress['queue_done'] = float(counter[0])
counter[0] += 1
self.update_current_tasks()
self.update_global_progress()
time.sleep(0.1)
if self.interrupt or not self.job_status_running:
raise InterruptException()
except ProcessException as pe:
logging.error(pe.message)
return False
except InterruptException as i:
raise i
except PydioSdkDefaultException as p:
raise p
except Exception as ex:
logging.exception(ex.message)
return False
return True
try:
if sys.platform.startswith('win'):
self.marked_for_snapshot_pathes = list(
set(self.current_store.find_modified_parents()) -
set(self.marked_for_snapshot_pathes))
self.current_store.process_changes_with_callback(
processor_callback)
except InterruptException as iexc:
pass
logger.log_state(
_('%i files modified') %
self.global_progress['queue_done'], 'success')
if self.global_progress['queue_done']:
logger.log_notif(
_('%i files modified') %
self.global_progress['queue_done'], 'success')
except PydioSdkDefaultException as re:
logging.error(re.message)
logger.log_state(re.message, 'error')
except SSLError as rt:
logging.error(rt.message)
logger.log_state(
_('An SSL error happened, please check the logs'), 'error')
except ProxyError as rt:
logging.error(rt.message)
logger.log_state(
_('A proxy error happened, please check the logs'),
'error')
except TooManyRedirects as rt:
logging.error(rt.message)
logger.log_state(_('Connection error: too many redirects'),
'error')
except ChunkedEncodingError as rt:
logging.error(rt.message)
logger.log_state(
_('Chunked encoding error, please check the logs'),
'error')
except ContentDecodingError as rt:
logging.error(rt.message)
logger.log_state(
_('Content Decoding error, please check the logs'),
'error')
except InvalidSchema as rt:
logging.error(rt.message)
logger.log_state(_('Http connection error: invalid schema.'),
'error')
except InvalidURL as rt:
logging.error(rt.message)
logger.log_state(_('Http connection error: invalid URL.'),
'error')
except Timeout as to:
logging.error(to)
logger.log_state(_('Connection timeout, will retry later.'),
'error')
except RequestException as ree:
logging.error(ree.message)
logger.log_state(_('Cannot resolve domain!'), 'error')
except Exception as e:
if not (e.message.lower().count('[quota limit reached]')
or e.message.lower().count('[file permissions]')):
logging.exception('Unexpected Error: %s' % e.message)
logger.log_state(
_('Unexpected Error: %s') % e.message, 'error')
logging.debug('Finished this cycle, waiting for %i seconds' %
self.online_timer)
self.exit_loop_clean(logger)
very_first = False
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser('Pydio Synchronization Tool')
# Pass a server configuration via arguments
parser.add_argument('-s', '--server', help='Server URL, with http(s) and path to pydio', type=unicode,
default='http://localhost')
parser.add_argument('-d', '--directory', help='Local directory', type=unicode, default=None)
parser.add_argument('-w', '--workspace', help='Id or Alias of workspace to synchronize', type=unicode, default=None)
parser.add_argument('-r', '--remote_folder', help='Path to an existing folder of the workspace to synchronize',
type=unicode, default=None)
parser.add_argument('-u', '--user', help='User name', type=unicode, default=None)
parser.add_argument('-p', '--password', help='Password', type=unicode, default=None)
parser.add_argument('-px', '--proxy', help='Enter like http::username::password::proxyIP::proxyPort::...::check_proxy_flag '
'By default proxy connection test happens, to avoid mention 0 or False', type=unicode, default=None)
parser.add_argument('-mp', '--memory_profile', help="To Generate the memory profile :: use <<-mp True >> as argument",
type=unicode, default=False)
parser.add_argument('-dir', '--direction', help='Synchro Direction', type=str, default='bi')
# Pass a configuration file
parser.add_argument('-f', '--file', type=unicode, help='Json file containing jobs configurations')
# Pass a path to rdiff binary
parser.add_argument('-i', '--rdiff', type=unicode, help='Path to rdiff executable', default=None)
# Configure API access
parser.add_argument('--api_user', help='Set the agent API username (instead of random)', type=unicode, default=None)
parser.add_argument('--api_password', help='Set the agent API password (instead of random)', type=unicode, default=None)
parser.add_argument('--api_address', help='Set the agent IP address. By default, no address means that local '
'access only is allowed.', type=str, default=None)
parser.add_argument('--api_port', help='Set the agent port. By default, will try to use 5556, and if not '
'available will switch to another port.', type=int, default=5556)
parser.add_argument('--diag', help='Run self diagnostic', action='store_true', default=False)
parser.add_argument('--diag-http', help='Check server connection', action='store_true', default=False)
parser.add_argument('--diag-imports', help='Check imports and exit', action='store_true', default=False)
parser.add_argument('--save-cfg', action='store_true', default=True)
parser.add_argument('--extract_html', help='Utils for extracting HTML strings and compiling po files to json',
type=unicode, default=False)
parser.add_argument('--auto-start', action='store_true')
parser.add_argument('-v', '--verbose', action='count', default=1)
args, _ = parser.parse_known_args(argv)
jobs_root_path = Path(__file__).parent / 'data'
if not jobs_root_path.exists():
jobs_root_path = Path(DEFAULT_DATA_PATH.encode(guess_filesystemencoding()))
if not jobs_root_path.exists():
jobs_root_path.mkdir(parents=True)
# This is a first start
user_dir = unicode(get_user_home(APP_NAME))
if not os.path.exists(user_dir):
try:
os.mkdir(user_dir)
except Exception:
pass
if os.path.exists(user_dir):
from pydio.utils.favorites_manager import add_to_favorites
add_to_favorites(user_dir, APP_NAME)
setup_logging(args.verbose, jobs_root_path)
if args.auto_start:
import pydio.autostart
pydio.autostart.setup(argv)
return 0
u_jobs_root_path = str(jobs_root_path).decode(guess_filesystemencoding())
config_manager = ConfigManager.Instance(configs_path=u_jobs_root_path, data_path=DEFAULT_PARENT_PATH)
jobs_loader = JobsLoader.Instance(data_path=u_jobs_root_path)
config_manager.set_rdiff_path(args.rdiff)
if args.proxy is not None:
data = args.proxy.split('::') if len(args.proxy.split('::'))%5 in range(0, 2) else logging.error("Wrong number of parameters pased for proxy")
msg = {}
for i in range(len(args.proxy.split('::'))/5):
msg[data[i*5]] = {"username": data[i*5+1], "password": data[i*5+2], "hostname": data[i*5+3], "port": data[i*5+4]}
proxy_flag = data[-1] if len(args.proxy.split('::'))%5 == 1 else True # default true
config_manager.set_user_proxy(msg, check_proxy_flag=proxy_flag)
return 0
if args.server and args.directory and args.workspace:
job_config = JobConfig()
job_config.load_from_cliargs(args)
data = {job_config.id: job_config}
if args.save_cfg:
logging.info("Storing config in %s", os.path.join(u_jobs_root_path, 'configs.json'))
jobs_loader.save_jobs(data)
else:
fp = args.file
if fp and fp != '.':
logging.info("Loading config from %s", fp)
jobs_loader.config_file = fp
jobs_loader.load_config()
data = jobs_loader.get_jobs()
logging.debug("data: %s" % json.dumps(data, default=JobConfig.encoder, indent=2))
if args.diag_imports:
# nothing more to do
return sys.exit(0)
if args.memory_profile:
from pydio.utils.pydio_profiler import LogFile
sys.stdout = LogFile('stdout')
if args.extract_html:
from pydio.utils.i18n import PoProcessor
proc = PoProcessor()
if args.extract_html == 'extract':
root = Path(__file__).parent
count = proc.extract_all_html_strings(str(root / 'ui' / 'res' ), str(root / 'res' / 'i18n' / 'html_strings.py' ))
logging.info('Wrote %i strings to html_strings.py - Now update PO files using standard tools' % count)
# nothing more to do
elif args.extract_html == 'compile':
root = Path(__file__).parent
proc.po_to_json(str(root / 'res' / 'i18n' / '*.po'), str(root / 'ui' / 'res' / 'i18n.js'))
return sys.exit(0)
if args.diag_http:
keys = data.keys()
if args.password:
smoke_tests = PydioDiagnostics(
data[keys[0]].server, data[keys[0]].workspace, data[keys[0]].remote_folder, data[keys[0]].user_id,
args.password)
else:
smoke_tests = PydioDiagnostics(
data[keys[0]].server, data[keys[0]].workspace, data[keys[0]].remote_folder, data[keys[0]].user_id)
rc = smoke_tests.run()
if rc != 0:
logging.error("Diagnostics failed: %s %s" % (str(rc), smoke_tests.status_message))
return sys.exit(rc)
ports_detector = PortsDetector(store_file=str(jobs_root_path / 'ports_config'), username=args.api_user,
password=args.api_password, default_port=args.api_port)
ports_detector.create_config_file()
scheduler = PydioScheduler.Instance(jobs_root_path=jobs_root_path, jobs_loader=jobs_loader)
server = PydioApi(ports_detector.get_port(), ports_detector.get_username(),
ports_detector.get_password(), external_ip=args.api_address)
from pydio.job import manager
manager.api_server = server
try:
thread.start_new_thread(server.start_server, ())
time.sleep(0.3)
if not server.running:
logging.error('Cannot start web server, exiting application')
sys.exit(1)
scheduler.start_all()
except (KeyboardInterrupt, SystemExit):
server.shutdown_server()
sys.exit()
def get(self):
"""
:return: {} containing some basic usage information
"""
jobs = JobsLoader.Instance().get_jobs()
resp = {
"errors": "zlib_blob",
"nberrors": 0,
"platform": platform.system()
}
for job_id in jobs:
resp[job_id] = {"nbsyncedfiles": 0, "lastseq": 0, "serverInfo": {}}
globalconfig = GlobalConfigManager.Instance(
configs_path=ConfigManager.Instance().get_configs_path())
resp["pydiosync_version"] = ConfigManager.Instance().get_version_data(
)["version"]
# parse logs for Errors, zip the errors
logdir = globalconfig.configs_path
files = os.listdir(logdir)
logfiles = []
for f in files:
if f.startswith(globalconfig.default_settings['log_configuration']
['log_file_name']):
logfiles.append(f)
compressor = zlib.compressobj()
compressed_data = ""
errors = "["
for logfile in logfiles:
try:
with open(os.path.join(logdir, logfile), 'r') as f:
for l in f.readlines():
if l.find('ERROR') > -1:
resp['nberrors'] += 1
errors += '"' + l.replace('\n', '') + '",'
compressed_data += compressor.compress(str(errors))
errors = ""
except Exception as e:
logging.exception(e)
compressor.compress("]")
compressed_data += compressor.flush()
# base64 encode the compressed extracted errors
resp['errors'] = compressed_data
resp["errors"] = base64.b64encode(resp["errors"])
# Instantiate and get logs from pydio.sqlite
for job_id in jobs:
try:
url = posixpath.join(jobs[job_id].server,
'index.php?get_action=get_boot_conf')
req = requests.get(url, verify=False)
logging.info("URL " + url)
logging.info(req.content)
jsonresp = json.loads(req.content)
if 'ajxpVersion' in jsonresp:
resp[job_id]['serverInfo']['ajxpVersion'] = jsonresp[
'ajxpVersion']
if 'customWording' in jsonresp:
resp[job_id]['serverInfo']['customWording'] = jsonresp[
'customWording']
if 'currentLanguage' in jsonresp:
resp[job_id]['serverInfo']['currentLanguage'] = jsonresp[
'currentLanguage']
if 'theme' in jsonresp:
resp[job_id]['serverInfo']['theme'] = jsonresp['theme']
if 'licence_features' in jsonresp:
resp[job_id]['serverInfo']['licence_features'] = jsonresp[
'licence_features']
except Exception as e:
logging.exception(e)
pydiosqlite = SqlEventHandler(
includes=jobs[job_id].filters['includes'],
excludes=jobs[job_id].filters['excludes'],
basepath=jobs[job_id].directory,
job_data_path=os.path.join(globalconfig.configs_path, job_id))
dbstats = pydiosqlite.db_stats()
resp[job_id] = {}
resp[job_id]['nbsyncedfiles'] = dbstats['nbfiles']
resp[job_id]['nbdirs'] = dbstats['nbdirs']
#logging.info(dir(jobs[jobs.keys()[0]]))
try:
with open(
os.path.join(globalconfig.configs_path, job_id,
"sequence"), "rb") as f:
sequences = pickle.load(f)
resp[job_id]['lastseq'] = sequences['local']
resp[job_id]['remotelastseq'] = sequences['remote']
except Exception:
logging.info('Problem loading sequences file')
resp[job_id]['lastseq'] = -1
resp[job_id]['remotelastseq'] = -1
return resp